framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,96,1,128,1,float16,float16,0,125.134375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,96,2,128,1,float16,float16,0,124.932470703125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,96,4,128,1,float16,float16,0,125.5951904296875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,96,96,128,1,float16,float16,0,132.62215576171874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,96,96,128,1,float16,float16,0,65.92476806640624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,96,1,128,1,float16,float16,0,62.911614990234376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,96,1,128,1,float16,float16,0,249.9552490234375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,96,8,128,1,float16,float16,0,125.8919189453125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,96,2,128,1,float16,float16,0,63.1626220703125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,96,96,128,1,float16,float16,0,33.284732055664065
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,96,1,128,1,float16,float16,0,31.498077392578125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,96,2,128,1,float16,float16,0,249.4859619140625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,96,2,128,1,float16,float16,0,31.785800170898437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,96,4,128,1,float16,float16,0,62.8039306640625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,96,8,128,1,float16,float16,0,248.348291015625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,96,4,128,1,float16,float16,0,254.0667724609375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,96,8,128,1,float16,float16,0,63.239111328125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,96,4,128,1,float16,float16,0,31.56806640625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,96,8,128,1,float16,float16,0,31.704019165039064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,96,1,128,1,float16,float16,0,71.34321899414063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,96,96,128,1,float16,float16,0,76.3310791015625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,96,2,128,1,float16,float16,0,71.03345947265625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,96,4,128,1,float16,float16,0,71.24000854492188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,96,1,128,1,float16,float16,0,35.62948303222656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,96,96,128,1,float16,float16,0,38.31429443359375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,96,2,128,1,float16,float16,0,35.67570190429687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,96,1,128,1,float16,float16,0,141.70123291015625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,96,2,128,1,float16,float16,0,143.2335205078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,96,96,128,1,float16,float16,0,19.0802001953125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,96,8,128,1,float16,float16,0,141.01680908203124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,96,4,128,1,float16,float16,0,142.0034912109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,96,8,128,1,float16,float16,0,71.7657958984375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,96,1,128,1,float16,float16,0,17.81190185546875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,96,4,128,1,float16,float16,0,35.691409301757815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,96,2,128,1,float16,float16,0,18.16325988769531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,96,8,128,1,float16,float16,0,36.170065307617186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,96,8,128,1,float16,float16,0,17.744207763671874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,96,4,128,1,float16,float16,0,17.938327026367187
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,96,1,128,1,float16,float16,0,50.25184020996094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,96,96,128,1,float16,float16,0,53.62062377929688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,96,2,128,1,float16,float16,0,49.14132080078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,96,4,128,1,float16,float16,0,49.7733642578125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,96,1,128,1,float16,float16,0,24.627113342285156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,96,2,128,1,float16,float16,0,24.941712951660158
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,96,96,128,1,float16,float16,0,26.947933959960938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,96,1,128,1,float16,float16,0,98.91343383789062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,96,96,128,1,float16,float16,0,13.631402587890625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,96,2,128,1,float16,float16,0,99.78431396484375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,96,4,128,1,float16,float16,0,99.14698486328125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,96,8,128,1,float16,float16,0,99.55449829101562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,96,1,128,1,float16,float16,0,12.517497253417968
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,96,8,128,1,float16,float16,0,50.56568603515625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,96,4,128,1,float16,float16,0,24.716351318359376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,96,8,128,1,float16,float16,0,25.051010131835938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,96,2,128,1,float16,float16,0,12.642489624023437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,96,4,128,1,float16,float16,0,12.625529479980468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,96,8,128,1,float16,float16,0,12.593041229248048
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,96,1,128,1,float16,float16,0,64.6527099609375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,96,2,128,1,float16,float16,0,64.40807495117187
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,96,4,128,1,float16,float16,0,63.918572998046876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,96,96,128,1,float16,float16,0,69.036083984375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,96,1,128,1,float16,float16,0,31.907760620117188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,96,2,128,1,float16,float16,0,31.973507690429688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,96,96,128,1,float16,float16,0,34.975588989257815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,96,1,128,1,float16,float16,0,127.72373046875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,96,2,128,1,float16,float16,0,127.66846923828125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,96,96,128,1,float16,float16,0,17.520220947265624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,96,8,128,1,float16,float16,0,126.76895751953126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,96,4,128,1,float16,float16,0,129.8536865234375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,96,4,128,1,float16,float16,0,31.996316528320314
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,96,8,128,1,float16,float16,0,65.092919921875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,96,8,128,1,float16,float16,0,32.149234008789065
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,96,96,128,1,float16,float16,0,8.897777557373047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,96,1,128,1,float16,float16,0,16.02012481689453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,96,2,128,1,float16,float16,0,16.015087890625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,96,4,128,1,float16,float16,0,16.22368469238281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,96,1,128,1,float16,float16,0,8.097457885742188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,96,2,128,1,float16,float16,0,8.213533020019531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,96,8,128,1,float16,float16,0,16.01573791503906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,96,4,128,1,float16,float16,0,8.089899444580078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,96,8,128,1,float16,float16,0,8.194608306884765
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,96,1,128,1,float16,float16,0,35.846273803710936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,96,2,128,1,float16,float16,0,36.34534912109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,96,96,128,1,float16,float16,0,40.4564697265625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,96,4,128,1,float16,float16,0,36.95809326171875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,96,96,128,1,float16,float16,0,20.011032104492188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,96,1,128,1,float16,float16,0,18.429127502441407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,96,2,128,1,float16,float16,0,18.40293731689453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,96,1,128,1,float16,float16,0,72.75892944335938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,96,2,128,1,float16,float16,0,72.81193237304687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,96,4,128,1,float16,float16,0,73.34896850585938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,96,96,128,1,float16,float16,0,10.222806549072265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,96,8,128,1,float16,float16,0,72.91920166015625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,96,8,128,1,float16,float16,0,36.11338500976562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,96,4,128,1,float16,float16,0,18.271871948242186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,96,96,128,1,float16,float16,0,5.095254516601562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,96,1,128,1,float16,float16,0,9.118307495117188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,96,8,128,1,float16,float16,0,18.626416015625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,96,2,128,1,float16,float16,0,9.114891052246094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,96,4,128,1,float16,float16,0,9.296006774902343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,96,8,128,1,float16,float16,0,9.399568176269531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,96,1,128,1,float16,float16,0,4.526332855224609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,96,2,128,1,float16,float16,0,4.54072151184082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,96,4,128,1,float16,float16,0,4.5831249237060545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,96,8,128,1,float16,float16,0,4.6323295593261715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,96,1,128,1,float16,float16,0,32.89394836425781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,96,2,128,1,float16,float16,0,33.07766418457031
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,96,96,128,1,float16,float16,0,37.63697509765625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,96,4,128,1,float16,float16,0,33.13507080078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,96,96,128,1,float16,float16,0,18.854542541503907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,96,1,128,1,float16,float16,0,16.493733215332032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,96,2,128,1,float16,float16,0,16.659150695800783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,96,2,128,1,float16,float16,0,65.99109497070313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,96,1,128,1,float16,float16,0,66.62631225585938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,96,8,128,1,float16,float16,0,65.58003540039063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,96,4,128,1,float16,float16,0,67.05067749023438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,96,8,128,1,float16,float16,0,33.18743896484375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,96,96,128,1,float16,float16,0,9.525918579101562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,96,4,128,1,float16,float16,0,16.706414794921876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,96,8,128,1,float16,float16,0,16.720278930664062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,96,1,128,1,float16,float16,0,8.274423980712891
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,96,2,128,1,float16,float16,0,8.39194107055664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,96,96,128,1,float16,float16,0,4.745806503295898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,96,1,128,1,float16,float16,0,4.113731384277344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,96,4,128,1,float16,float16,0,8.193566131591798
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,96,8,128,1,float16,float16,0,8.514459228515625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,96,2,128,1,float16,float16,0,4.1293182373046875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,96,96,128,1,float16,float16,0,2.4180383682250977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,96,1,128,1,float16,float16,0,2.078068733215332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,96,2,128,1,float16,float16,0,2.118788719177246
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,96,4,128,1,float16,float16,0,4.149070358276367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,96,4,128,1,float16,float16,0,2.0988544464111327
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,96,8,128,1,float16,float16,0,4.199686431884766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,96,8,128,1,float16,float16,0,2.1302223205566406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,96,1,128,1,float16,float16,0,18.575999450683593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,96,2,128,1,float16,float16,0,18.84285125732422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,96,4,128,1,float16,float16,0,19.1513427734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,96,96,128,1,float16,float16,0,22.344549560546874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,96,1,128,1,float16,float16,0,9.619454193115235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,96,96,128,1,float16,float16,0,11.23197250366211
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,96,2,128,1,float16,float16,0,9.487973022460938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,96,1,128,1,float16,float16,0,37.7966552734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,96,4,128,1,float16,float16,0,37.916107177734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,96,2,128,1,float16,float16,0,38.302703857421875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,96,8,128,1,float16,float16,0,18.902925109863283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,96,8,128,1,float16,float16,0,38.403134155273435
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,96,96,128,1,float16,float16,0,5.590516662597656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,96,4,128,1,float16,float16,0,9.739566040039062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,96,8,128,1,float16,float16,0,9.595758056640625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,96,1,128,1,float16,float16,0,2.334284782409668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,96,96,128,1,float16,float16,0,2.830281639099121
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,96,1,128,1,float16,float16,0,4.727883148193359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,96,2,128,1,float16,float16,0,2.38775520324707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,96,2,128,1,float16,float16,0,4.686590576171875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,96,8,128,1,float16,float16,0,4.750564956665039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,96,96,128,1,float16,float16,0,1.458409595489502
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,96,4,128,1,float16,float16,0,4.815055847167969
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,96,1,128,1,float16,float16,0,1.220088005065918
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,96,4,128,1,float16,float16,0,2.394588851928711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,96,8,128,1,float16,float16,0,2.402102470397949
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,96,2,128,1,float16,float16,0,1.2476559638977052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,96,4,128,1,float16,float16,0,1.2394255638122558
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,96,8,128,1,float16,float16,0,1.2590751647949219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,96,1,128,1,float16,float16,0,17.551609802246094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,96,2,128,1,float16,float16,0,17.413597106933594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,96,4,128,1,float16,float16,0,17.205712890625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,96,96,128,1,float16,float16,0,21.899368286132812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,96,1,128,1,float16,float16,0,8.534210968017579
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,96,96,128,1,float16,float16,0,10.945047760009766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,96,2,128,1,float16,float16,0,8.749793243408202
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,96,1,128,1,float16,float16,0,34.90673828125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,96,2,128,1,float16,float16,0,35.33281555175781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,96,4,128,1,float16,float16,0,35.213070678710935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,96,8,128,1,float16,float16,0,35.46932678222656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,96,8,128,1,float16,float16,0,17.985383605957033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,96,4,128,1,float16,float16,0,8.73326416015625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,96,96,128,1,float16,float16,0,5.489947128295898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,96,1,128,1,float16,float16,0,2.1453760147094725
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,96,8,128,1,float16,float16,0,8.819071960449218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,96,1,128,1,float16,float16,0,4.329036712646484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,96,96,128,1,float16,float16,0,2.7796192169189453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,96,2,128,1,float16,float16,0,2.1537904739379883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,96,2,128,1,float16,float16,0,4.391430282592774
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,96,4,128,1,float16,float16,0,4.35986557006836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,96,8,128,1,float16,float16,0,4.416105651855469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,96,1,128,1,float16,float16,0,1.1116415977478027
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,96,96,128,1,float16,float16,0,1.4091456413269043
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,96,4,128,1,float16,float16,0,2.1866336822509767
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,96,2,128,1,float16,float16,0,1.1054096221923828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,96,96,128,1,float16,float16,0,0.7566832065582275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,96,4,128,1,float16,float16,0,1.151908779144287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,96,8,128,1,float16,float16,0,1.1476719856262207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,96,1,128,1,float16,float16,0,0.5685584068298339
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,96,8,128,1,float16,float16,0,2.194460868835449
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,96,2,128,1,float16,float16,0,0.5948463916778565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,96,4,128,1,float16,float16,0,0.6039023876190186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,96,8,128,1,float16,float16,0,0.6021391868591308
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,96,2,128,1,float16,float16,0,9.965558624267578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,96,1,128,1,float16,float16,0,10.137644958496093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,96,4,128,1,float16,float16,0,10.233360290527344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,96,96,128,1,float16,float16,0,13.445681762695312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,96,96,128,1,float16,float16,0,6.744563293457031
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,96,1,128,1,float16,float16,0,5.014380645751953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,96,1,128,1,float16,float16,0,20.260389709472655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,96,4,128,1,float16,float16,0,20.4684326171875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,96,2,128,1,float16,float16,0,20.608717346191405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,96,8,128,1,float16,float16,0,10.17944793701172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,96,8,128,1,float16,float16,0,20.870193481445312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,96,2,128,1,float16,float16,0,5.066262435913086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,96,1,128,1,float16,float16,0,2.5032703399658205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,96,2,128,1,float16,float16,0,2.474527931213379
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,96,4,128,1,float16,float16,0,2.5484432220458983
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,96,4,128,1,float16,float16,0,5.103211212158203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,96,96,128,1,float16,float16,0,3.364672088623047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,96,1,128,1,float16,float16,0,1.25174560546875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,96,2,128,1,float16,float16,0,1.3063632011413575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,96,8,128,1,float16,float16,0,2.576767921447754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,96,8,128,1,float16,float16,0,5.138446426391601
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,96,96,128,1,float16,float16,0,1.7352319717407227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,96,4,128,1,float16,float16,0,1.3024000167846679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,96,8,128,1,float16,float16,0,1.3162960052490233
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,96,1,128,1,float16,float16,0,0.6491648197174072
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,96,2,128,1,float16,float16,0,0.6731776237487793
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,96,4,128,1,float16,float16,0,0.698140811920166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,96,96,128,1,float16,float16,0,0.8601344108581543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,96,1,128,1,float16,float16,0,0.3544687986373901
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,96,2,128,1,float16,float16,0,0.37773919105529785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,96,4,128,1,float16,float16,0,0.36726720333099366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,96,8,128,1,float16,float16,0,0.6898335933685302
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,96,8,128,1,float16,float16,0,0.3812704086303711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,96,96,128,1,float16,float16,0,0.47812161445617674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,96,1,128,1,float16,float16,0,9.639151763916015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,96,2,128,1,float16,float16,0,9.762007904052734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,96,4,128,1,float16,float16,0,9.799027252197266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,96,96,128,1,float16,float16,0,13.917604064941406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,96,96,128,1,float16,float16,0,7.108198547363282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,96,1,128,1,float16,float16,0,4.834892654418946
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,96,1,128,1,float16,float16,0,19.347799682617186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,96,2,128,1,float16,float16,0,19.290080261230468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,96,4,128,1,float16,float16,0,19.459275817871095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,96,2,128,1,float16,float16,0,4.721462249755859
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,96,8,128,1,float16,float16,0,19.80475769042969
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,96,8,128,1,float16,float16,0,9.92079849243164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,96,1,128,1,float16,float16,0,2.328771209716797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,96,96,128,1,float16,float16,0,1.7646144866943358
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,96,2,128,1,float16,float16,0,2.3459087371826173
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,96,4,128,1,float16,float16,0,2.3805871963500977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,96,4,128,1,float16,float16,0,4.869516754150391
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,96,8,128,1,float16,float16,0,2.4664943695068358
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,96,96,128,1,float16,float16,0,3.50029296875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,96,8,128,1,float16,float16,0,4.982825469970703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,96,1,128,1,float16,float16,0,1.2156687736511231
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,96,2,128,1,float16,float16,0,1.191152000427246
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,96,1,128,1,float16,float16,0,0.6148064136505127
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,96,4,128,1,float16,float16,0,1.220580768585205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,96,96,128,1,float16,float16,0,0.9376399993896485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,96,8,128,1,float16,float16,0,1.2284480094909669
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,96,2,128,1,float16,float16,0,0.6187024116516113
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,96,1,128,1,float16,float16,0,0.32536160945892334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,96,4,128,1,float16,float16,0,0.6462224006652832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,96,96,128,1,float16,float16,0,0.261900806427002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,96,2,128,1,float16,float16,0,0.33903040885925295
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,96,8,128,1,float16,float16,0,0.3458048105239868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,96,96,128,1,float16,float16,0,0.48267521858215334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,96,8,128,1,float16,float16,0,0.6420112133026123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,96,1,128,1,float16,float16,0,0.18116639852523803
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,96,2,128,1,float16,float16,0,0.18351680040359497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,96,8,128,1,float16,float16,0,0.198470401763916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,96,4,128,1,float16,float16,0,0.19220000505447388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,96,4,128,1,float16,float16,0,0.34118080139160156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,96,1,128,1,float16,float16,0,5.667152023315429
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,96,2,128,1,float16,float16,0,5.777862548828125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,96,4,128,1,float16,float16,0,5.8658702850341795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,96,96,128,1,float16,float16,0,10.227334594726562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,96,1,128,1,float16,float16,0,11.335897827148438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,96,2,128,1,float16,float16,0,11.566790771484374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,96,4,128,1,float16,float16,0,11.742940521240234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,96,8,128,1,float16,float16,0,12.051889801025391
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,96,1,128,1,float16,float16,0,2.8571264266967775
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,96,8,128,1,float16,float16,0,5.988497543334961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,96,2,128,1,float16,float16,0,2.848673629760742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,96,1,128,1,float16,float16,0,1.4012255668640137
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,96,2,128,1,float16,float16,0,1.4267600059509278
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,96,4,128,1,float16,float16,0,1.4304559707641602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,96,4,128,1,float16,float16,0,2.838131141662598
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,96,8,128,1,float16,float16,0,2.941200065612793
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,96,96,128,1,float16,float16,0,2.551945686340332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,96,1,128,1,float16,float16,0,0.7323440074920654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,96,2,128,1,float16,float16,0,0.7220208168029785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,96,96,128,1,float16,float16,0,5.117599868774414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,96,4,128,1,float16,float16,0,0.7499072074890136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,96,8,128,1,float16,float16,0,1.5003472328186036
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,96,96,128,1,float16,float16,0,1.3063695907592774
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,96,2,128,1,float16,float16,0,0.37362558841705323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,96,1,128,1,float16,float16,0,0.3716288089752197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,96,8,128,1,float16,float16,0,0.7590847969055176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,96,96,128,1,float16,float16,0,0.6708352088928222
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,96,4,128,1,float16,float16,0,0.3753632068634033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,96,8,128,1,float16,float16,0,0.3891871929168701
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,96,96,128,1,float16,float16,0,0.19339679479598998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,96,1,128,1,float16,float16,0,0.11212639808654785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,96,8,128,1,float16,float16,0,0.21317760944366454
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,96,1,128,1,float16,float16,0,0.19869120121002198
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,96,96,128,1,float16,float16,0,0.3613264083862305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,96,4,128,1,float16,float16,0,0.11877919435501098
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,96,1,128,1,float16,float16,0,0.06793280243873596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,96,96,128,1,float16,float16,0,0.1095088005065918
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,96,2,128,1,float16,float16,0,0.07239999771118164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,96,4,128,1,float16,float16,0,0.07185119986534119
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,96,8,128,1,float16,float16,0,0.0780896008014679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,96,4,128,1,float16,float16,0,0.20392959117889403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,96,2,128,1,float16,float16,0,0.20162560939788818
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,96,8,128,1,float16,float16,0,0.12295039892196655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,96,1,128,1,float16,float16,0,3.7841758728027344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,96,2,128,1,float16,float16,0,3.8196990966796873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,96,4,128,1,float16,float16,0,3.878462219238281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,96,1,128,1,float16,float16,0,1.8072128295898438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,96,2,128,1,float16,float16,0,1.8844640731811524
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,96,2,128,1,float16,float16,0,0.11741600036621094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,96,4,128,1,float16,float16,0,1.9221887588500977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,96,8,128,1,float16,float16,0,1.983171272277832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,96,8,128,1,float16,float16,0,4.012542343139648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,96,1,128,1,float16,float16,0,0.9468288421630859
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,96,96,128,1,float16,float16,0,4.2716320037841795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,96,2,128,1,float16,float16,0,0.9102640151977539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,96,4,128,1,float16,float16,0,0.9861488342285156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,96,8,128,1,float16,float16,0,1.0049920082092285
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,96,96,128,1,float16,float16,0,2.1746992111206054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,96,2,128,1,float16,float16,0,0.47711682319641113
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,96,4,128,1,float16,float16,0,0.47724318504333496
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,96,96,128,1,float16,float16,0,1.1371376037597656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,96,8,128,1,float16,float16,0,0.5112224102020264
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,96,96,128,1,float16,float16,0,0.5741744041442871
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,96,4,128,1,float16,float16,0,0.250548791885376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,96,1,128,1,float16,float16,0,0.24688959121704102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,96,2,128,1,float16,float16,0,0.25083839893341064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,96,1,128,1,float16,float16,0,0.13777120113372804
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,96,96,128,1,float16,float16,0,0.3054368019104004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,96,8,128,1,float16,float16,0,0.2695823907852173
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,96,2,128,1,float16,float16,0,0.13677599430084228
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,96,4,128,1,float16,float16,0,0.1394144058227539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,96,1,128,1,float16,float16,0,0.08243680000305176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,96,8,128,1,float16,float16,0,0.14502079486846925
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,96,2,128,1,float16,float16,0,0.08294720053672791
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,96,4,128,1,float16,float16,0,0.08180959820747376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,96,96,128,1,float16,float16,0,0.16781920194625854
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,96,8,128,1,float16,float16,0,0.08530719876289368
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,96,96,128,1,float16,float16,0,0.08984000086784363
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,96,1,128,1,float16,float16,0,0.05103520154953003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,96,4,128,1,float16,float16,0,0.05036960244178772
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,96,2,128,1,float16,float16,0,0.05047680139541626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,96,1,128,1,float16,float16,0,0.028678399324417115
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,96,8,128,1,float16,float16,0,0.053921598196029666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,96,4,128,1,float16,float16,0,0.030244800448417663
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,96,2,128,1,float16,float16,0,0.0317984014749527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,96,8,128,1,float16,float16,0,0.028867200016975403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,96,1,128,1,float16,float16,0,0.4773536205291748
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,96,1,128,1,float16,float16,0,1.3685376167297363
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,96,4,128,1,float16,float16,0,1.4292767524719239
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,96,2,128,1,float16,float16,0,1.3965727806091308
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,96,8,128,1,float16,float16,0,1.539742374420166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,96,1,128,1,float16,float16,0,0.6915440082550048
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,96,96,128,1,float16,float16,0,2.0888544082641602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,96,4,128,1,float16,float16,0,0.7504735946655273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,96,1,128,1,float16,float16,0,0.3594559907913208
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,96,8,128,1,float16,float16,0,0.7820112228393554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,96,2,128,1,float16,float16,0,0.3626528024673462
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,96,4,128,1,float16,float16,0,0.3794559955596924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,96,8,128,1,float16,float16,0,0.40925917625427244
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,96,1,128,1,float16,float16,0,0.19093120098114014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,96,96,128,1,float16,float16,0,0.05294399857521057
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,96,2,128,1,float16,float16,0,0.19101760387420655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,96,4,128,1,float16,float16,0,0.19916319847106934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,96,96,128,1,float16,float16,0,0.5579055786132813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,96,1,128,1,float16,float16,0,0.10430400371551514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,96,8,128,1,float16,float16,0,0.2139888048171997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,96,2,128,1,float16,float16,0,0.10609279870986939
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,96,96,128,1,float16,float16,0,0.2906960010528564
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,96,4,128,1,float16,float16,0,0.10942399501800537
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,96,8,128,1,float16,float16,0,0.11886080503463745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,96,96,128,1,float16,float16,0,0.1544319987297058
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,96,1,128,1,float16,float16,0,0.06518880128860474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,96,4,128,1,float16,float16,0,0.06644319891929626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,96,2,128,1,float16,float16,0,0.06458719968795776
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,96,1,128,1,float16,float16,0,0.03516480028629303
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,96,8,128,1,float16,float16,0,0.0692031979560852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,96,2,128,1,float16,float16,0,0.03710240125656128
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,96,96,128,1,float16,float16,0,0.08345760107040405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,96,96,128,1,float16,float16,0,0.04386880099773407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,96,8,128,1,float16,float16,0,0.04063679873943329
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,96,2,128,1,float16,float16,0,0.023878400027751923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,96,1,128,1,float16,float16,0,0.023403200507164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,96,4,128,1,float16,float16,0,0.022228799760341644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,96,8,128,1,float16,float16,0,0.0236175999045372
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,96,1,128,1,float16,float16,0,0.018904000520706177
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,96,96,128,1,float16,float16,0,0.024433599412441255
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,96,2,128,1,float16,float16,0,0.018423999845981597
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,96,8,128,1,float16,float16,0,0.018688000738620758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,96,1,128,1,float16,float16,0,0.6639999866485595
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,96,2,128,1,float16,float16,0,0.6968592166900635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,96,4,128,1,float16,float16,0,0.7636832237243653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,96,8,128,1,float16,float16,0,0.8168704032897949
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,96,2,128,1,float16,float16,0,0.698414421081543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,96,1,128,1,float16,float16,0,0.3521663904190063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,96,2,128,1,float16,float16,0,0.362771201133728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,96,96,128,1,float16,float16,0,1.0727999687194825
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,96,4,128,1,float16,float16,0,0.3906752109527588
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,96,96,128,1,float16,float16,0,1.0686944007873536
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,96,8,128,1,float16,float16,0,0.4267615795135498
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,96,1,128,1,float16,float16,0,0.18107199668884277
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,96,2,128,1,float16,float16,0,0.18808480501174926
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,96,96,128,1,float16,float16,0,0.5444064140319824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,96,4,128,1,float16,float16,0,0.2045680046081543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,96,8,128,1,float16,float16,0,0.21824800968170166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,96,1,128,1,float16,float16,0,0.0980288028717041
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,96,4,128,1,float16,float16,0,0.11027040481567382
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,96,4,128,1,float16,float16,0,0.03783040046691895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,96,2,128,1,float16,float16,0,0.10474079847335815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,96,96,128,1,float16,float16,0,0.1541152000427246
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,96,2,128,1,float16,float16,0,0.06047840118408203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,96,1,128,1,float16,float16,0,0.06138560175895691
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,96,8,128,1,float16,float16,0,0.06545439958572388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,96,4,128,1,float16,float16,0,0.06311360001564026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,96,4,128,1,float16,float16,0,0.018963199853897095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,96,2,128,1,float16,float16,0,0.03154079914093018
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,96,96,128,1,float16,float16,0,0.08521599769592285
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,96,1,128,1,float16,float16,0,0.031488001346588135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,96,8,128,1,float16,float16,0,0.034836798906326294
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,96,96,128,1,float16,float16,0,0.046121600270271304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,96,1,128,1,float16,float16,0,0.020423999428749083
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,96,2,128,1,float16,float16,0,0.020681600272655486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,96,4,128,1,float16,float16,0,0.02011840045452118
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,96,8,128,1,float16,float16,0,0.020129600167274476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,96,96,128,1,float16,float16,0,0.022991999983787537
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,96,1,128,1,float16,float16,0,0.016993600130081176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,96,2,128,1,float16,float16,0,0.016811199486255646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,96,4,128,1,float16,float16,0,0.01653279960155487
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,96,8,128,1,float16,float16,0,0.01676799952983856
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,96,96,128,1,float16,float16,0,0.018483200669288637
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,96,1,128,1,float16,float16,0,0.01544640064239502
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,96,2,128,1,float16,float16,0,0.01541920006275177
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,96,4,128,1,float16,float16,0,0.015537600219249725
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,96,8,128,1,float16,float16,0,0.015481600165367126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,96,1,128,1,float16,float16,0,0.328766393661499
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,96,2,128,1,float16,float16,0,0.3416800022125244
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,96,4,128,1,float16,float16,0,0.3605263948440552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,96,8,128,1,float16,float16,0,0.386411190032959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,96,1,128,1,float16,float16,0,0.17589279413223266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,96,2,128,1,float16,float16,0,0.18102719783782958
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,96,4,128,1,float16,float16,0,0.19344799518585204
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,96,8,128,1,float16,float16,0,0.20597119331359864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,96,96,128,1,float16,float16,0,0.6980991840362549
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,96,1,128,1,float16,float16,0,0.10039199590682983
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,96,96,128,1,float16,float16,0,0.35938560962677
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,96,2,128,1,float16,float16,0,0.10085439682006836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,96,4,128,1,float16,float16,0,0.10431360006332398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,96,8,128,1,float16,float16,0,0.11238559484481811
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,96,1,128,1,float16,float16,0,0.05965920090675354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,96,96,128,1,float16,float16,0,0.18562239408493042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,96,4,128,1,float16,float16,0,0.062431997060775755
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,96,8,128,1,float16,float16,0,0.0648959994316101
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,96,1,128,1,float16,float16,0,0.030854400992393494
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,96,96,128,1,float16,float16,0,0.10030879974365234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,96,96,128,1,float16,float16,0,0.286243200302124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,96,2,128,1,float16,float16,0,0.031065601110458373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,96,4,128,1,float16,float16,0,0.032278400659561154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,96,8,128,1,float16,float16,0,0.03502880036830902
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,96,1,128,1,float16,float16,0,0.019833600521087645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,96,96,128,1,float16,float16,0,0.0579584002494812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,96,8,128,1,float16,float16,0,0.1151360034942627
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,96,2,128,1,float16,float16,0,0.019912000000476836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,96,4,128,1,float16,float16,0,0.019920000433921815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,96,8,128,1,float16,float16,0,0.019729599356651306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,96,96,128,1,float16,float16,0,0.030164799094200133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,96,2,128,1,float16,float16,0,0.01632000058889389
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,96,1,128,1,float16,float16,0,0.016564799845218657
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,96,8,128,1,float16,float16,0,0.016439999639987945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,96,96,128,1,float16,float16,0,0.020182399451732634
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,96,1,128,1,float16,float16,0,0.015507200360298156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,96,2,128,1,float16,float16,0,0.015091200172901154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,96,4,128,1,float16,float16,0,0.015270400047302245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,96,8,128,1,float16,float16,0,0.015265600383281707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,96,4,128,1,float16,float16,0,0.031692799925804135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,96,96,128,1,float16,float16,0,0.017888000607490538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,96,1,128,1,float16,float16,0,0.01480959951877594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,96,2,128,1,float16,float16,0,0.015022400021553039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,96,4,128,1,float16,float16,0,0.014905600249767304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,96,8,128,1,float16,float16,0,0.01520639955997467
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,96,1,128,1,float16,float16,0,0.17276320457458497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,96,2,128,1,float16,float16,0,0.17483999729156494
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,96,4,128,1,float16,float16,0,0.18889600038528442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,96,1,128,1,float16,float16,0,0.09904639720916748
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,96,8,128,1,float16,float16,0,0.2263040065765381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,96,2,128,1,float16,float16,0,0.0998304009437561
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,96,96,128,1,float16,float16,0,0.5624911785125732
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,96,4,128,1,float16,float16,0,0.10551999807357788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,96,8,128,1,float16,float16,0,0.1227455973625183
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,96,1,128,1,float16,float16,0,0.061244797706604
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,96,8,128,1,float16,float16,0,0.07167519927024842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,96,4,128,1,float16,float16,0,0.062480002641677856
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,96,2,128,1,float16,float16,0,0.06244959831237793
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,96,1,128,1,float16,float16,0,0.031220799684524535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,96,96,128,1,float16,float16,0,0.15258560180664063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,96,96,128,1,float16,float16,0,0.29069440364837645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,96,2,128,1,float16,float16,0,0.0314848005771637
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,96,4,128,1,float16,float16,0,0.0325872004032135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,96,8,128,1,float16,float16,0,0.04156480133533478
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,96,1,128,1,float16,float16,0,0.019732800126075745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,96,2,128,1,float16,float16,0,0.01979839950799942
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,96,4,128,1,float16,float16,0,0.019758400321006776
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,96,96,128,1,float16,float16,0,0.08453919887542724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,96,8,128,1,float16,float16,0,0.02476000040769577
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,96,1,128,1,float16,float16,0,0.016572800278663636
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,96,4,128,1,float16,float16,0,0.015982399880886077
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,96,2,128,1,float16,float16,0,0.016371199488639833
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,96,8,128,1,float16,float16,0,0.018262399733066557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,96,96,128,1,float16,float16,0,0.02739039957523346
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,96,1,128,1,float16,float16,0,0.015175999701023101
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,96,2,128,1,float16,float16,0,0.015156799554824829
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,96,4,128,1,float16,float16,0,0.014990399777889251
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,96,8,128,1,float16,float16,0,0.015390400588512421
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,96,96,128,1,float16,float16,0,0.019582399725914003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,96,1,128,1,float16,float16,0,0.014878399670124054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,96,2,128,1,float16,float16,0,0.014983999729156493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,96,4,128,1,float16,float16,0,0.015063999593257904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,96,8,128,1,float16,float16,0,0.015027199685573579
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,96,96,128,1,float16,float16,0,0.017499199509620665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,96,1,128,1,float16,float16,0,0.014689600467681885
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,96,2,128,1,float16,float16,0,0.014769600331783294
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,96,4,128,1,float16,float16,0,0.01470080018043518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,96,8,128,1,float16,float16,0,0.014913600683212281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,96,1,128,1,float16,float16,0,0.028798401355743408
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,96,2,128,1,float16,float16,0,0.03502399921417236
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,96,4,128,1,float16,float16,0,0.05235360264778137
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,96,1,128,1,float16,float16,0,0.019595199823379518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,96,8,128,1,float16,float16,0,0.08972160220146179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,96,2,128,1,float16,float16,0,0.025271999835968017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,96,4,128,1,float16,float16,0,0.016551999747753142
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,96,4,128,1,float16,float16,0,0.03267039954662323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,96,8,128,1,float16,float16,0,0.050014400482177736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,96,96,128,1,float16,float16,0,0.43216958045959475
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,96,1,128,1,float16,float16,0,0.01619359999895096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,96,2,128,1,float16,float16,0,0.017870399355888366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,96,4,128,1,float16,float16,0,0.023614400625228883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,96,8,128,1,float16,float16,0,0.03127520084381104
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,96,1,128,1,float16,float16,0,0.015184000134468079
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,96,96,128,1,float16,float16,0,0.11938879489898682
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,96,4,128,1,float16,float16,0,0.016894400119781494
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,96,8,128,1,float16,float16,0,0.02284799963235855
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,96,1,128,1,float16,float16,0,0.01454399973154068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,96,96,128,1,float16,float16,0,0.06736800074577332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,96,2,128,1,float16,float16,0,0.014425599575042724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,96,4,128,1,float16,float16,0,0.01424960047006607
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,96,8,128,1,float16,float16,0,0.016364799439907075
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,96,96,128,1,float16,float16,0,0.040755200386047366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,96,1,128,1,float16,float16,0,0.01451680064201355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,96,2,128,1,float16,float16,0,0.014115199446678162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,96,4,128,1,float16,float16,0,0.014363199472427368
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,96,8,128,1,float16,float16,0,0.014419199526309967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,96,96,128,1,float16,float16,0,0.043331199884414674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,96,96,128,1,float16,float16,0,0.026203200221061707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,96,8,128,1,float16,float16,0,0.014209599792957306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,96,4,128,1,float16,float16,0,0.01430879980325699
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,96,2,128,1,float16,float16,0,0.01417279988527298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,96,96,128,1,float16,float16,0,0.01876160055398941
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,96,1,128,1,float16,float16,0,0.014211200177669525
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,96,4,128,1,float16,float16,0,0.014022399485111237
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,96,2,128,1,float16,float16,0,0.014028799533843995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,96,96,128,1,float16,float16,0,0.016791999340057373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,96,1,128,1,float16,float16,0,0.014150400459766389
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,96,2,128,1,float16,float16,0,0.06114559769630432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,96,2,128,1,float16,float16,0,0.013894400000572205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,96,4,128,1,float16,float16,0,0.01390720009803772
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,96,8,128,1,float16,float16,0,0.014025600254535675
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,96,96,128,1,float16,float16,0,0.22488799095153808
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,96,2,128,1,float16,float16,0,0.015169599652290344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,96,1,128,1,float16,float16,0,0.014267200231552124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,96,8,128,1,float16,float16,0,0.01422560065984726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,64,2,128,1,float16,float16,0,84.26942138671875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,64,4,128,1,float16,float16,0,84.87904052734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,64,1,128,1,float16,float16,0,85.42403564453124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,64,1,128,1,float16,float16,0,42.906411743164064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,64,8,128,1,float16,float16,0,166.53680419921875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,64,2,128,1,float16,float16,0,167.96431884765624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,64,4,128,1,float16,float16,0,168.19871826171874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,64,1,128,1,float16,float16,0,170.2842529296875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,64,8,128,1,float16,float16,0,84.80371704101563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,64,2,128,1,float16,float16,0,42.88984680175781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,64,1,128,1,float16,float16,0,21.646543884277342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,64,2,128,1,float16,float16,0,21.225341796875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,64,4,128,1,float16,float16,0,21.651528930664064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,64,4,128,1,float16,float16,0,41.793157958984374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,64,8,128,1,float16,float16,0,42.47384338378906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,64,8,128,1,float16,float16,0,21.563609313964843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,64,64,128,1,float16,float16,0,22.30177459716797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,64,1,128,1,float16,float16,0,95.46067504882812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,64,2,128,1,float16,float16,0,96.62314453125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,64,64,128,1,float16,float16,0,44.624874877929685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,64,64,128,1,float16,float16,0,51.05843811035156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,64,4,128,1,float16,float16,0,94.25431518554687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,64,8,128,1,float16,float16,0,95.68447875976562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,64,1,128,1,float16,float16,0,47.48984375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,64,64,128,1,float16,float16,0,25.71552734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,64,1,128,1,float16,float16,0,23.935081481933594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,64,2,128,1,float16,float16,0,23.667599487304688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,64,64,128,1,float16,float16,0,88.2738037109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,64,2,128,1,float16,float16,0,47.860025024414064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,64,4,128,1,float16,float16,0,48.55140380859375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,64,64,128,1,float16,float16,0,12.85682373046875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,64,8,128,1,float16,float16,0,48.35049743652344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,64,4,128,1,float16,float16,0,23.900189208984376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,64,1,128,1,float16,float16,0,11.908656311035156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,64,2,128,1,float16,float16,0,12.02426528930664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,64,4,128,1,float16,float16,0,12.120913696289062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,64,8,128,1,float16,float16,0,24.382905578613283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,64,8,128,1,float16,float16,0,12.339395141601562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,64,1,128,1,float16,float16,0,33.44166259765625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,64,64,128,1,float16,float16,0,35.51705322265625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,64,2,128,1,float16,float16,0,33.7608642578125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,64,4,128,1,float16,float16,0,33.93413391113281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,64,64,128,1,float16,float16,0,17.8761962890625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,64,1,128,1,float16,float16,0,16.95979461669922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,64,2,128,1,float16,float16,0,17.00799560546875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,64,1,128,1,float16,float16,0,66.33577270507813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,64,2,128,1,float16,float16,0,66.9947021484375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,64,4,128,1,float16,float16,0,66.26751098632812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,64,8,128,1,float16,float16,0,65.98626708984375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,64,64,128,1,float16,float16,0,9.196723175048827
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,64,1,128,1,float16,float16,0,8.37597427368164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,64,8,128,1,float16,float16,0,33.649270629882814
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,64,2,128,1,float16,float16,0,8.441051483154297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,64,4,128,1,float16,float16,0,16.694259643554688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,64,8,128,1,float16,float16,0,17.035845947265624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,64,4,128,1,float16,float16,0,8.409286499023438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,64,8,128,1,float16,float16,0,8.446470642089844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,64,2,128,1,float16,float16,0,42.646270751953125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,64,1,128,1,float16,float16,0,43.397589111328124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,64,4,128,1,float16,float16,0,42.42770690917969
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,64,64,128,1,float16,float16,0,46.09751892089844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,64,1,128,1,float16,float16,0,21.15857696533203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,64,64,128,1,float16,float16,0,23.427911376953126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,64,2,128,1,float16,float16,0,21.371273803710938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,64,64,128,1,float16,float16,0,11.703520202636719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,64,1,128,1,float16,float16,0,86.78858642578125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,64,2,128,1,float16,float16,0,85.34902954101562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,64,4,128,1,float16,float16,0,86.0919921875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,64,8,128,1,float16,float16,0,86.73709106445312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,64,8,128,1,float16,float16,0,43.03564453125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,64,4,128,1,float16,float16,0,21.230715942382812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,64,8,128,1,float16,float16,0,21.94114532470703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,64,1,128,1,float16,float16,0,10.817228698730469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,64,64,128,1,float16,float16,0,5.94239501953125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,64,1,128,1,float16,float16,0,5.3698478698730465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,64,2,128,1,float16,float16,0,5.361318588256836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,64,2,128,1,float16,float16,0,11.009139251708984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,64,4,128,1,float16,float16,0,10.817708587646484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,64,8,128,1,float16,float16,0,10.933934020996094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,64,4,128,1,float16,float16,0,5.508668899536133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,64,8,128,1,float16,float16,0,5.506167984008789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,64,1,128,1,float16,float16,0,24.472418212890624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,64,2,128,1,float16,float16,0,24.707978820800783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,64,64,128,1,float16,float16,0,26.70234069824219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,64,4,128,1,float16,float16,0,24.47145233154297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,64,1,128,1,float16,float16,0,12.192323303222656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,64,2,128,1,float16,float16,0,12.330977630615234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,64,64,128,1,float16,float16,0,13.622445678710937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,64,2,128,1,float16,float16,0,48.45789794921875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,64,1,128,1,float16,float16,0,48.931155395507815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,64,64,128,1,float16,float16,0,6.805178833007813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,64,4,128,1,float16,float16,0,48.220147705078126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,64,8,128,1,float16,float16,0,49.76540832519531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,64,8,128,1,float16,float16,0,24.672384643554686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,64,4,128,1,float16,float16,0,12.2675537109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,64,8,128,1,float16,float16,0,12.282788848876953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,64,1,128,1,float16,float16,0,6.089396667480469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,64,2,128,1,float16,float16,0,6.19873275756836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,64,64,128,1,float16,float16,0,3.516257476806641
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,64,1,128,1,float16,float16,0,3.094513511657715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,64,4,128,1,float16,float16,0,6.212364959716797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,64,8,128,1,float16,float16,0,6.159228897094726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,64,2,128,1,float16,float16,0,3.087268829345703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,64,4,128,1,float16,float16,0,3.137558364868164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,64,8,128,1,float16,float16,0,3.137918472290039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,64,1,128,1,float16,float16,0,22.052545166015626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,64,2,128,1,float16,float16,0,22.13507537841797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,64,4,128,1,float16,float16,0,22.11041259765625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,64,64,128,1,float16,float16,0,24.88056640625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,64,1,128,1,float16,float16,0,10.967110443115235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,64,2,128,1,float16,float16,0,10.8609375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,64,64,128,1,float16,float16,0,12.589409637451173
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,64,1,128,1,float16,float16,0,44.28626708984375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,64,64,128,1,float16,float16,0,6.301462554931641
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,64,4,128,1,float16,float16,0,44.41541748046875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,64,2,128,1,float16,float16,0,44.723330688476565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,64,8,128,1,float16,float16,0,45.20400695800781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,64,8,128,1,float16,float16,0,22.338851928710938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,64,4,128,1,float16,float16,0,11.05896453857422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,64,8,128,1,float16,float16,0,11.062118530273438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,64,64,128,1,float16,float16,0,3.201839828491211
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,64,1,128,1,float16,float16,0,2.7566959381103517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,64,1,128,1,float16,float16,0,5.5253551483154295
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,64,2,128,1,float16,float16,0,2.803873634338379
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,64,2,128,1,float16,float16,0,5.517982482910156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,64,4,128,1,float16,float16,0,5.579260635375976
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,64,8,128,1,float16,float16,0,5.6958465576171875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,64,4,128,1,float16,float16,0,2.7817264556884767
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,64,64,128,1,float16,float16,0,1.6741247177124023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,64,1,128,1,float16,float16,0,1.3990575790405273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,64,2,128,1,float16,float16,0,1.4354111671447753
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,64,8,128,1,float16,float16,0,2.870916748046875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,64,4,128,1,float16,float16,0,1.478649616241455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,64,8,128,1,float16,float16,0,1.5268863677978515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,64,1,128,1,float16,float16,0,12.657695770263672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,64,2,128,1,float16,float16,0,12.861766052246093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,64,64,128,1,float16,float16,0,15.049053955078126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,64,4,128,1,float16,float16,0,12.71278076171875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,64,64,128,1,float16,float16,0,7.568701171875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,64,1,128,1,float16,float16,0,6.373548889160157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,64,1,128,1,float16,float16,0,24.939920043945314
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,64,2,128,1,float16,float16,0,25.466427612304688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,64,4,128,1,float16,float16,0,25.40440216064453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,64,8,128,1,float16,float16,0,25.69764099121094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,64,2,128,1,float16,float16,0,6.320111846923828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,64,8,128,1,float16,float16,0,12.926612854003906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,64,64,128,1,float16,float16,0,3.7164894104003907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,64,1,128,1,float16,float16,0,3.1643503189086912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,64,2,128,1,float16,float16,0,3.1390096664428713
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,64,4,128,1,float16,float16,0,6.490214538574219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,64,8,128,1,float16,float16,0,6.53106689453125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,64,4,128,1,float16,float16,0,3.2218048095703127
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,64,64,128,1,float16,float16,0,1.8900463104248046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,64,1,128,1,float16,float16,0,1.5669903755187988
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,64,8,128,1,float16,float16,0,3.2281150817871094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,64,1,128,1,float16,float16,0,0.8574687957763671
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,64,2,128,1,float16,float16,0,0.8496368408203125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,64,2,128,1,float16,float16,0,1.6030799865722656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,64,4,128,1,float16,float16,0,1.6250192642211914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,64,8,128,1,float16,float16,0,1.667598342895508
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,64,4,128,1,float16,float16,0,0.8480719566345215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,64,8,128,1,float16,float16,0,0.8930352210998536
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,64,64,128,1,float16,float16,0,1.0214768409729005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,64,1,128,1,float16,float16,0,11.420706939697265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,64,2,128,1,float16,float16,0,11.666665649414062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,64,4,128,1,float16,float16,0,11.814328002929688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,64,64,128,1,float16,float16,0,14.774888610839843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,64,64,128,1,float16,float16,0,7.269541168212891
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,64,1,128,1,float16,float16,0,5.818471908569336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,64,2,128,1,float16,float16,0,5.900136184692383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,64,1,128,1,float16,float16,0,23.201353454589842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,64,2,128,1,float16,float16,0,23.315953063964844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,64,8,128,1,float16,float16,0,11.75308609008789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,64,4,128,1,float16,float16,0,23.675531005859376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,64,8,128,1,float16,float16,0,23.772550964355467
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,64,64,128,1,float16,float16,0,3.684734344482422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,64,1,128,1,float16,float16,0,2.8989536285400392
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,64,4,128,1,float16,float16,0,5.865396881103516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,64,64,128,1,float16,float16,0,1.8275823593139648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,64,2,128,1,float16,float16,0,2.9310895919799806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,64,4,128,1,float16,float16,0,2.901313591003418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,64,8,128,1,float16,float16,0,6.0098625183105465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,64,1,128,1,float16,float16,0,1.486131191253662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,64,8,128,1,float16,float16,0,2.9876304626464845
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,64,64,128,1,float16,float16,0,0.9732640266418457
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,64,2,128,1,float16,float16,0,1.482414436340332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,64,1,128,1,float16,float16,0,0.7571216106414795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,64,8,128,1,float16,float16,0,1.4937135696411132
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,64,4,128,1,float16,float16,0,1.4843711853027344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,64,2,128,1,float16,float16,0,0.7912928104400635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,64,64,128,1,float16,float16,0,0.5256944179534913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,64,4,128,1,float16,float16,0,0.8026896476745605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,64,8,128,1,float16,float16,0,0.8025103569030761
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,64,1,128,1,float16,float16,0,0.4153632164001465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,64,2,128,1,float16,float16,0,0.4173151969909668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,64,4,128,1,float16,float16,0,0.43543357849121095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,64,8,128,1,float16,float16,0,0.44104480743408203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,64,1,128,1,float16,float16,0,6.669264221191407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,64,2,128,1,float16,float16,0,6.795359802246094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,64,4,128,1,float16,float16,0,6.827283477783203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,64,64,128,1,float16,float16,0,9.054183959960938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,64,1,128,1,float16,float16,0,3.349630355834961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,64,64,128,1,float16,float16,0,4.4571678161621096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,64,2,128,1,float16,float16,0,3.4173648834228514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,64,1,128,1,float16,float16,0,13.793931579589843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,64,2,128,1,float16,float16,0,13.760539245605468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,64,4,128,1,float16,float16,0,13.93199462890625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,64,8,128,1,float16,float16,0,6.852559661865234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,64,8,128,1,float16,float16,0,14.051206970214844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,64,4,128,1,float16,float16,0,3.417675018310547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,64,1,128,1,float16,float16,0,1.7108879089355469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,64,8,128,1,float16,float16,0,3.457126235961914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,64,2,128,1,float16,float16,0,1.704350471496582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,64,64,128,1,float16,float16,0,1.1698351860046388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,64,64,128,1,float16,float16,0,2.2437471389770507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,64,4,128,1,float16,float16,0,1.7339008331298829
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,64,8,128,1,float16,float16,0,1.7337823867797852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,64,1,128,1,float16,float16,0,0.451796817779541
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,64,64,128,1,float16,float16,0,0.6077424049377441
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,64,2,128,1,float16,float16,0,0.8891407966613769
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,64,4,128,1,float16,float16,0,0.9062911987304687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,64,8,128,1,float16,float16,0,0.9214608192443847
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,64,4,128,1,float16,float16,0,0.4701072216033936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,64,2,128,1,float16,float16,0,0.47803521156311035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,64,1,128,1,float16,float16,0,0.25475358963012695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,64,8,128,1,float16,float16,0,0.47904157638549805
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,64,2,128,1,float16,float16,0,0.24694879055023194
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,64,64,128,1,float16,float16,0,0.3348160028457642
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,64,4,128,1,float16,float16,0,0.2651711940765381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,64,8,128,1,float16,float16,0,0.27647039890289304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,64,1,128,1,float16,float16,0,0.8719792366027832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,64,1,128,1,float16,float16,0,6.528137969970703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,64,2,128,1,float16,float16,0,6.35847053527832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,64,64,128,1,float16,float16,0,9.215342712402343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,64,2,128,1,float16,float16,0,12.914369201660156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,64,4,128,1,float16,float16,0,13.063645935058593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,64,1,128,1,float16,float16,0,13.008453369140625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,64,1,128,1,float16,float16,0,3.1404415130615235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,64,64,128,1,float16,float16,0,4.595382308959961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,64,8,128,1,float16,float16,0,13.5712646484375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,64,4,128,1,float16,float16,0,6.600542449951172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,64,8,128,1,float16,float16,0,6.581208038330078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,64,1,128,1,float16,float16,0,1.5675616264343262
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,64,2,128,1,float16,float16,0,1.5786864280700683
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,64,4,128,1,float16,float16,0,1.6341983795166015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,64,64,128,1,float16,float16,0,2.3105951309204102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,64,2,128,1,float16,float16,0,3.155108833312988
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,64,4,128,1,float16,float16,0,3.202854537963867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,64,1,128,1,float16,float16,0,0.8096495628356933
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,64,2,128,1,float16,float16,0,0.8541711807250977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,64,8,128,1,float16,float16,0,1.665825653076172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,64,8,128,1,float16,float16,0,3.3013584136962892
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,64,64,128,1,float16,float16,0,1.1839167594909668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,64,4,128,1,float16,float16,0,0.8273488044738769
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,64,1,128,1,float16,float16,0,0.42116479873657225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,64,64,128,1,float16,float16,0,0.6395936012268066
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,64,1,128,1,float16,float16,0,0.23106880187988282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,64,64,128,1,float16,float16,0,0.3341952085494995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,64,8,128,1,float16,float16,0,0.8480992317199707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,64,4,128,1,float16,float16,0,0.4458032131195068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,64,2,128,1,float16,float16,0,0.2375216007232666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,64,4,128,1,float16,float16,0,0.24864161014556885
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,64,8,128,1,float16,float16,0,0.245963191986084
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,64,64,128,1,float16,float16,0,0.18873440027236937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,64,2,128,1,float16,float16,0,0.14341280460357667
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,64,1,128,1,float16,float16,0,0.14258400201797486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,64,8,128,1,float16,float16,0,0.14867520332336426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,64,2,128,1,float16,float16,0,0.43962559700012205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,64,8,128,1,float16,float16,0,0.47748961448669436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,64,4,128,1,float16,float16,0,0.1369312047958374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,64,1,128,1,float16,float16,0,3.823076629638672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,64,64,128,1,float16,float16,0,6.712910461425781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,64,2,128,1,float16,float16,0,3.838129425048828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,64,1,128,1,float16,float16,0,7.561420440673828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,64,4,128,1,float16,float16,0,3.9254608154296875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,64,2,128,1,float16,float16,0,7.735756683349609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,64,4,128,1,float16,float16,0,7.883707427978516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,64,8,128,1,float16,float16,0,8.242017364501953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,64,1,128,1,float16,float16,0,1.9086671829223634
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,64,1,128,1,float16,float16,0,0.9580623626708984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,64,2,128,1,float16,float16,0,1.8988048553466796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,64,4,128,1,float16,float16,0,1.9604400634765624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,64,64,128,1,float16,float16,0,1.7071968078613282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,64,8,128,1,float16,float16,0,2.00533447265625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,64,64,128,1,float16,float16,0,3.3897167205810548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,64,2,128,1,float16,float16,0,0.9593423843383789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,64,1,128,1,float16,float16,0,0.49283361434936523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,64,2,128,1,float16,float16,0,0.5190688133239746
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,64,8,128,1,float16,float16,0,1.01692476272583
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,64,4,128,1,float16,float16,0,0.9719679832458497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,64,64,128,1,float16,float16,0,0.8884592056274414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,64,8,128,1,float16,float16,0,4.119163131713867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,64,64,128,1,float16,float16,0,0.4563568115234375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,64,8,128,1,float16,float16,0,0.5139311790466309
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,64,1,128,1,float16,float16,0,0.2632384061813354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,64,4,128,1,float16,float16,0,0.5046495914459228
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,64,8,128,1,float16,float16,0,0.2890127897262573
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,64,64,128,1,float16,float16,0,0.25176799297332764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,64,1,128,1,float16,float16,0,0.1448192000389099
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,64,2,128,1,float16,float16,0,0.08693119883537292
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,64,2,128,1,float16,float16,0,0.14868639707565307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,64,4,128,1,float16,float16,0,0.1500975966453552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,64,64,128,1,float16,float16,0,0.14277600049972533
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,64,4,128,1,float16,float16,0,0.09420480132102967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,64,8,128,1,float16,float16,0,0.0931280016899109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,64,64,128,1,float16,float16,0,0.08265439867973327
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,64,2,128,1,float16,float16,0,0.052532798051834105
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,64,4,128,1,float16,float16,0,0.05286880135536194
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,64,8,128,1,float16,float16,0,0.05722079873085022
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,64,1,128,1,float16,float16,0,2.47979679107666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,64,2,128,1,float16,float16,0,0.26684958934783937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,64,2,128,1,float16,float16,0,2.52052001953125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,64,4,128,1,float16,float16,0,2.508815956115723
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,64,8,128,1,float16,float16,0,0.1543023943901062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,64,1,128,1,float16,float16,0,0.06210079789161682
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,64,1,128,1,float16,float16,0,1.2758416175842284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,64,2,128,1,float16,float16,0,1.2642671585083007
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,64,4,128,1,float16,float16,0,1.2675600051879883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,64,8,128,1,float16,float16,0,1.3378064155578613
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,64,1,128,1,float16,float16,0,0.6359024047851562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,64,8,128,1,float16,float16,0,2.7626016616821287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,64,2,128,1,float16,float16,0,0.63264479637146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,64,4,128,1,float16,float16,0,0.6449984073638916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,64,64,128,1,float16,float16,0,2.8247535705566404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,64,4,128,1,float16,float16,0,0.2730576038360596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,64,64,128,1,float16,float16,0,1.45031681060791
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,64,1,128,1,float16,float16,0,0.32187039852142335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,64,8,128,1,float16,float16,0,0.686414384841919
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,64,1,128,1,float16,float16,0,0.08946719765663147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,64,64,128,1,float16,float16,0,0.7514527797698974
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,64,2,128,1,float16,float16,0,0.32710719108581543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,64,1,128,1,float16,float16,0,0.17491840124130248
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,64,2,128,1,float16,float16,0,0.17479679584503174
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,64,8,128,1,float16,float16,0,0.35531039237976075
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,64,4,128,1,float16,float16,0,0.18395999670028687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,64,64,128,1,float16,float16,0,0.38345119953155515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,64,8,128,1,float16,float16,0,0.18974560499191284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,64,1,128,1,float16,float16,0,0.09896000027656555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,64,64,128,1,float16,float16,0,0.20686240196228028
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,64,2,128,1,float16,float16,0,0.10155999660491943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,64,8,128,1,float16,float16,0,0.1077455997467041
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,64,1,128,1,float16,float16,0,0.06028320193290711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,64,64,128,1,float16,float16,0,0.11755679845809937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,64,2,128,1,float16,float16,0,0.0634447991847992
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,64,4,128,1,float16,float16,0,0.06420959830284119
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,64,64,128,1,float16,float16,0,0.06559039950370789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,64,8,128,1,float16,float16,0,0.0658240020275116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,64,1,128,1,float16,float16,0,0.041356799006462094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,64,64,128,1,float16,float16,0,0.039259201288223265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,64,2,128,1,float16,float16,0,0.04061599969863892
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,64,1,128,1,float16,float16,0,0.0262719988822937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,64,2,128,1,float16,float16,0,0.028041601181030273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,64,4,128,1,float16,float16,0,0.029902398586273193
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,64,8,128,1,float16,float16,0,0.02926880121231079
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,64,1,128,1,float16,float16,0,0.9192496299743652
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,64,2,128,1,float16,float16,0,0.9497967720031738
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,64,4,128,1,float16,float16,0,1.0019167900085448
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,64,4,128,1,float16,float16,0,0.33350720405578616
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,64,1,128,1,float16,float16,0,0.4751488208770752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,64,4,128,1,float16,float16,0,0.1039471983909607
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,64,8,128,1,float16,float16,0,1.1248463630676269
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,64,4,128,1,float16,float16,0,0.03981919884681702
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,64,8,128,1,float16,float16,0,0.03867200016975403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,64,64,128,1,float16,float16,0,1.3887040138244628
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,64,2,128,1,float16,float16,0,0.25313758850097656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,64,2,128,1,float16,float16,0,0.48293437957763674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,64,1,128,1,float16,float16,0,0.2461632013320923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,64,8,128,1,float16,float16,0,0.5649759769439697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,64,4,128,1,float16,float16,0,0.5122144222259521
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,64,4,128,1,float16,float16,0,0.2718559980392456
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,64,64,128,1,float16,float16,0,0.709662389755249
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,64,1,128,1,float16,float16,0,0.13345439434051515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,64,8,128,1,float16,float16,0,0.29776160717010497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,64,2,128,1,float16,float16,0,0.1366703987121582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,64,4,128,1,float16,float16,0,0.14578720331192016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,64,8,128,1,float16,float16,0,0.16397119760513307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,64,64,128,1,float16,float16,0,0.36424961090087893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,64,64,128,1,float16,float16,0,0.20072479248046876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,64,1,128,1,float16,float16,0,0.0782480001449585
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,64,4,128,1,float16,float16,0,0.0820527970790863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,64,8,128,1,float16,float16,0,0.09016960263252258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,64,2,128,1,float16,float16,0,0.07899839878082275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,64,4,128,1,float16,float16,0,0.051999998092651364
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,64,1,128,1,float16,float16,0,0.05071200132369995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,64,64,128,1,float16,float16,0,0.10723199844360351
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,64,8,128,1,float16,float16,0,0.05539360046386719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,64,1,128,1,float16,float16,0,0.02991360127925873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,64,2,128,1,float16,float16,0,0.02953760027885437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,64,4,128,1,float16,float16,0,0.029760000109672547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,64,64,128,1,float16,float16,0,0.030108800530433653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,64,1,128,1,float16,float16,0,0.021329599618911742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,64,2,128,1,float16,float16,0,0.022495999932289124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,64,4,128,1,float16,float16,0,0.022896000742912294
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,64,8,128,1,float16,float16,0,0.022177599370479584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,64,64,128,1,float16,float16,0,0.021300800144672394
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,64,1,128,1,float16,float16,0,0.01774719953536987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,64,2,128,1,float16,float16,0,0.01807679980993271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,64,4,128,1,float16,float16,0,0.018294399976730345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,64,8,128,1,float16,float16,0,0.01873439997434616
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,64,1,128,1,float16,float16,0,0.47420477867126465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,64,2,128,1,float16,float16,0,0.47930078506469725
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,64,4,128,1,float16,float16,0,0.5123856067657471
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,64,8,128,1,float16,float16,0,0.5876800060272217
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,64,2,128,1,float16,float16,0,0.252892804145813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,64,4,128,1,float16,float16,0,0.2683248043060303
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,64,64,128,1,float16,float16,0,0.7156496047973633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,64,2,128,1,float16,float16,0,0.13653440475463868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,64,8,128,1,float16,float16,0,0.3049936056137085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,64,1,128,1,float16,float16,0,0.1348047971725464
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,64,64,128,1,float16,float16,0,0.36871039867401123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,64,8,128,1,float16,float16,0,0.16262400150299072
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,64,4,128,1,float16,float16,0,0.1472591996192932
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,64,64,128,1,float16,float16,0,0.1960752010345459
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,64,2,128,1,float16,float16,0,0.07671840190887451
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,64,4,128,1,float16,float16,0,0.08221279978752136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,64,1,128,1,float16,float16,0,0.046697598695755
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,64,64,128,1,float16,float16,0,0.10657440423965454
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,64,2,128,1,float16,float16,0,0.051769602298736575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,64,2,128,1,float16,float16,0,0.04743840098381043
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,64,4,128,1,float16,float16,0,0.04890879988670349
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,64,64,128,1,float16,float16,0,0.05965120196342468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,64,8,128,1,float16,float16,0,0.051551997661590576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,64,8,128,1,float16,float16,0,0.030214399099349976
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,64,1,128,1,float16,float16,0,0.026726400852203368
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,64,2,128,1,float16,float16,0,0.027215999364852906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,64,4,128,1,float16,float16,0,0.02715040147304535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,64,8,128,1,float16,float16,0,0.028147199749946596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,64,64,128,1,float16,float16,0,0.03091840147972107
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,64,1,128,1,float16,float16,0,0.018535999953746794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,64,2,128,1,float16,float16,0,0.01852640062570572
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,64,4,128,1,float16,float16,0,0.018836799263954162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,64,8,128,1,float16,float16,0,0.019355200231075287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,64,64,128,1,float16,float16,0,0.02009119987487793
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,64,1,128,1,float16,float16,0,0.015937599539756774
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,64,2,128,1,float16,float16,0,0.0156016007065773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,64,4,128,1,float16,float16,0,0.01592320054769516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,64,8,128,1,float16,float16,0,0.016436800360679626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,64,64,128,1,float16,float16,0,0.016847999393939973
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,64,1,128,1,float16,float16,0,0.01536639928817749
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,64,4,128,1,float16,float16,0,0.01526080071926117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,64,8,128,1,float16,float16,0,0.015441599488258361
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,64,1,128,1,float16,float16,0,0.25174880027770996
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,64,1,128,1,float16,float16,0,0.2309727907180786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,64,2,128,1,float16,float16,0,0.24559199810028076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,64,4,128,1,float16,float16,0,0.2577343940734863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,64,8,128,1,float16,float16,0,0.2826607942581177
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,64,1,128,1,float16,float16,0,0.12763359546661376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,64,2,128,1,float16,float16,0,0.1321776032447815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,64,4,128,1,float16,float16,0,0.13816479444503785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,64,1,128,1,float16,float16,0,0.07582079768180847
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,64,64,128,1,float16,float16,0,0.47649121284484863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,64,64,128,1,float16,float16,0,0.24166719913482665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,64,1,128,1,float16,float16,0,0.07514560222625732
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,64,2,128,1,float16,float16,0,0.07824479937553405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,64,4,128,1,float16,float16,0,0.0794272005558014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,64,8,128,1,float16,float16,0,0.08881120085716247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,64,8,128,1,float16,float16,0,0.08709440231323243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,64,1,128,1,float16,float16,0,0.04735200107097626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,64,2,128,1,float16,float16,0,0.04822719991207123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,64,4,128,1,float16,float16,0,0.04931040108203888
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,64,8,128,1,float16,float16,0,0.052095997333526614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,64,64,128,1,float16,float16,0,0.05960000157356262
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,64,64,128,1,float16,float16,0,0.07094240188598633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,64,2,128,1,float16,float16,0,0.026716798543930054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,64,1,128,1,float16,float16,0,0.027241599559783936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,64,4,128,1,float16,float16,0,0.026742398738861084
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,64,8,128,1,float16,float16,0,0.02762719988822937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,64,64,128,1,float16,float16,0,0.03699679970741272
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,64,2,128,1,float16,float16,0,0.018268799781799315
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,64,1,128,1,float16,float16,0,0.018438400328159334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,64,4,128,1,float16,float16,0,0.018454399704933167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,64,8,128,1,float16,float16,0,0.01902240067720413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,64,64,128,1,float16,float16,0,0.025183999538421632
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,64,1,128,1,float16,float16,0,0.01564960032701492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,64,4,128,1,float16,float16,0,0.015428799390792846
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,64,8,128,1,float16,float16,0,0.016176000237464905
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,64,64,128,1,float16,float16,0,0.018783999979496
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,64,2,128,1,float16,float16,0,0.015033599734306336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,64,1,128,1,float16,float16,0,0.015080000460147857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,64,8,128,1,float16,float16,0,0.015211200714111328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,64,64,128,1,float16,float16,0,0.01656640022993088
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,64,4,128,1,float16,float16,0,0.015192000567913056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,64,1,128,1,float16,float16,0,0.014852799475193024
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,64,2,128,1,float16,float16,0,0.01475519984960556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,64,8,128,1,float16,float16,0,0.015067200362682342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,64,4,128,1,float16,float16,0,0.014975999295711518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,64,1,128,1,float16,float16,0,0.12556480169296264
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,64,2,128,1,float16,float16,0,0.1283344030380249
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,64,4,128,1,float16,float16,0,0.14005600214004515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,64,8,128,1,float16,float16,0,0.15012160539627076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,64,2,128,1,float16,float16,0,0.0763696014881134
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,64,1,128,1,float16,float16,0,0.07475039958953858
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,64,4,128,1,float16,float16,0,0.08017280101776122
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,64,8,128,1,float16,float16,0,0.08584480285644532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,64,8,128,1,float16,float16,0,0.15196000337600707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,64,64,128,1,float16,float16,0,0.19570080041885377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,64,4,128,1,float16,float16,0,0.0499455988407135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,64,2,128,1,float16,float16,0,0.04859200119972229
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,64,8,128,1,float16,float16,0,0.053014397621154785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,64,64,128,1,float16,float16,0,0.10576640367507935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,64,1,128,1,float16,float16,0,0.027353599667549133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,64,2,128,1,float16,float16,0,0.027401599287986755
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,64,4,128,1,float16,float16,0,0.02744640111923218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,64,64,128,1,float16,float16,0,0.13050880432128906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,64,8,128,1,float16,float16,0,0.028700798749923706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,64,64,128,1,float16,float16,0,0.05404000282287598
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,64,1,128,1,float16,float16,0,0.018438400328159334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,64,2,128,1,float16,float16,0,0.018084800243377684
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,64,4,128,1,float16,float16,0,0.018406400084495546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,64,8,128,1,float16,float16,0,0.019436800479888917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,64,64,128,1,float16,float16,0,0.03223679959774017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,64,1,128,1,float16,float16,0,0.015647999942302704
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,64,2,128,1,float16,float16,0,0.01531360000371933
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,64,4,128,1,float16,float16,0,0.01523520052433014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,64,8,128,1,float16,float16,0,0.0160848006606102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,64,2,128,1,float16,float16,0,0.015345600247383118
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,64,64,128,1,float16,float16,0,0.024344000220298766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,64,1,128,1,float16,float16,0,0.014876799285411834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,64,2,128,1,float16,float16,0,0.015006400644779205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,64,4,128,1,float16,float16,0,0.014950400590896607
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,64,8,128,1,float16,float16,0,0.015281599760055543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,64,64,128,1,float16,float16,0,0.018321600556373597
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,64,1,128,1,float16,float16,0,0.014936000108718872
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,64,2,128,1,float16,float16,0,0.014854399859905243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,64,4,128,1,float16,float16,0,0.015024000406265258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,64,2,128,1,float16,float16,0,0.015356799960136414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,64,64,128,1,float16,float16,0,0.016286399960517884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,64,1,128,1,float16,float16,0,0.014726400375366211
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,64,2,128,1,float16,float16,0,0.014819200336933135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,64,4,128,1,float16,float16,0,0.014851200580596923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,64,8,128,1,float16,float16,0,0.014843200147151948
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,64,1,128,1,float16,float16,0,0.02754719853401184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,64,2,128,1,float16,float16,0,0.03477280139923096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,64,4,128,1,float16,float16,0,0.052286398410797116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,64,8,128,1,float16,float16,0,0.08924800157546997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,64,1,128,1,float16,float16,0,0.019273599982261656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,64,2,128,1,float16,float16,0,0.024935999512672426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,64,4,128,1,float16,float16,0,0.03251520097255707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,64,64,128,1,float16,float16,0,0.29443519115447997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,64,8,128,1,float16,float16,0,0.05013120174407959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,64,1,128,1,float16,float16,0,0.016123199462890626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,64,2,128,1,float16,float16,0,0.01775519996881485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,64,64,128,1,float16,float16,0,0.15350879430770875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,64,8,128,1,float16,float16,0,0.03127039968967438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,64,4,128,1,float16,float16,0,0.02337439954280853
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,64,1,128,1,float16,float16,0,0.01507679969072342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,64,2,128,1,float16,float16,0,0.015270400047302245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,64,4,128,1,float16,float16,0,0.016910399496555328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,64,8,128,1,float16,float16,0,0.02274080067873001
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,64,64,128,1,float16,float16,0,0.3790672063827515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,64,64,128,1,float16,float16,0,0.04801760017871857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,64,1,128,1,float16,float16,0,0.014281600713729858
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,64,2,128,1,float16,float16,0,0.014407999813556671
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,64,8,128,1,float16,float16,0,0.016238400340080263
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,64,4,128,1,float16,float16,0,0.014455999433994293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,64,1,128,1,float16,float16,0,0.014363199472427368
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,64,2,128,1,float16,float16,0,0.014292800426483154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,64,4,128,1,float16,float16,0,0.014297600090503692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,64,8,128,1,float16,float16,0,0.014640000462532044
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,64,64,128,1,float16,float16,0,0.023319999873638152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,64,1,128,1,float16,float16,0,0.014097599685192109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,64,2,128,1,float16,float16,0,0.01435520052909851
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,64,1,128,1,float16,float16,0,0.048003199696540835
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,64,4,128,1,float16,float16,0,0.01420000046491623
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,64,8,128,1,float16,float16,0,0.014393599331378936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,64,64,128,1,float16,float16,0,0.01796479970216751
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,64,1,128,1,float16,float16,0,0.014009599387645722
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,64,2,128,1,float16,float16,0,0.014280000329017639
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,64,4,128,1,float16,float16,0,0.014033600687980652
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,64,8,128,1,float16,float16,0,0.014407999813556671
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,64,1,128,1,float16,float16,0,0.013996799290180207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,64,64,128,1,float16,float16,0,0.016033600270748138
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,64,2,128,1,float16,float16,0,0.013911999762058258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,64,4,128,1,float16,float16,0,0.014324800670146942
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,64,8,128,1,float16,float16,0,0.014056000113487243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,64,8,128,1,float16,float16,0,0.01496479958295822
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,64,64,128,1,float16,float16,0,0.08416000008583069
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,64,64,128,1,float16,float16,0,0.030663999915122985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,48,1,128,1,float16,float16,0,63.000433349609374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,48,2,128,1,float16,float16,0,63.762481689453125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,48,4,128,1,float16,float16,0,63.320709228515625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,48,1,128,1,float16,float16,0,31.775070190429688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,48,4,128,1,float16,float16,0,125.39720458984375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,48,8,128,1,float16,float16,0,127.78853759765624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,48,2,128,1,float16,float16,0,128.13848876953125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,48,1,128,1,float16,float16,0,128.42568359375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,48,8,128,1,float16,float16,0,63.5504150390625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,48,2,128,1,float16,float16,0,31.8310302734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,48,1,128,1,float16,float16,0,16.266639709472656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,48,2,128,1,float16,float16,0,15.950527954101563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,48,4,128,1,float16,float16,0,15.992256164550781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,48,4,128,1,float16,float16,0,31.557421875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,48,8,128,1,float16,float16,0,32.219183349609374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,48,8,128,1,float16,float16,0,16.38843536376953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,48,1,128,1,float16,float16,0,71.31058349609376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,48,2,128,1,float16,float16,0,71.379052734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,48,4,128,1,float16,float16,0,71.00575561523438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,48,8,128,1,float16,float16,0,72.35415649414062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,48,1,128,1,float16,float16,0,35.77400512695313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,48,2,128,1,float16,float16,0,35.86790161132812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,48,4,128,1,float16,float16,0,35.57358093261719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,48,48,128,1,float16,float16,0,16.913665771484375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,48,8,128,1,float16,float16,0,36.30494384765625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,48,48,128,1,float16,float16,0,19.083642578125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,48,1,128,1,float16,float16,0,17.996795654296875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,48,2,128,1,float16,float16,0,17.88234100341797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,48,48,128,1,float16,float16,0,33.62717895507812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,48,48,128,1,float16,float16,0,9.664456176757813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,48,1,128,1,float16,float16,0,9.082844543457032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,48,4,128,1,float16,float16,0,18.182740783691408
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,48,8,128,1,float16,float16,0,18.18406524658203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,48,48,128,1,float16,float16,0,38.50149841308594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,48,2,128,1,float16,float16,0,8.974251556396485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,48,4,128,1,float16,float16,0,9.066764831542969
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,48,8,128,1,float16,float16,0,9.247801971435546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,48,48,128,1,float16,float16,0,66.40650024414063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,48,48,128,1,float16,float16,0,26.452130126953126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,48,1,128,1,float16,float16,0,25.064892578125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,48,2,128,1,float16,float16,0,24.919889831542967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,48,1,128,1,float16,float16,0,12.535323333740234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,48,48,128,1,float16,float16,0,13.521644592285156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,48,1,128,1,float16,float16,0,49.81910400390625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,48,2,128,1,float16,float16,0,50.42399597167969
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,48,4,128,1,float16,float16,0,50.353857421875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,48,4,128,1,float16,float16,0,25.036798095703126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,48,8,128,1,float16,float16,0,51.11045532226562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,48,8,128,1,float16,float16,0,24.993919372558594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,48,2,128,1,float16,float16,0,12.535063934326171
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,48,48,128,1,float16,float16,0,6.847876739501953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,48,4,128,1,float16,float16,0,12.665548706054688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,48,1,128,1,float16,float16,0,6.348844909667969
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,48,2,128,1,float16,float16,0,6.479195404052734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,48,8,128,1,float16,float16,0,12.846992492675781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,48,4,128,1,float16,float16,0,6.2956687927246096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,48,8,128,1,float16,float16,0,6.400206756591797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,48,1,128,1,float16,float16,0,31.861239624023437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,48,2,128,1,float16,float16,0,31.875851440429688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,48,4,128,1,float16,float16,0,32.25255432128906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,48,48,128,1,float16,float16,0,35.07369384765625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,48,1,128,1,float16,float16,0,15.996232604980468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,48,48,128,1,float16,float16,0,17.594288635253907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,48,2,128,1,float16,float16,0,16.361068725585938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,48,1,128,1,float16,float16,0,64.465771484375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,48,2,128,1,float16,float16,0,64.33508911132813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,48,4,128,1,float16,float16,0,64.52828979492188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,48,48,128,1,float16,float16,0,8.883869171142578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,48,8,128,1,float16,float16,0,65.807958984375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,48,8,128,1,float16,float16,0,32.35582580566406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,48,4,128,1,float16,float16,0,16.183253479003906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,48,48,128,1,float16,float16,0,4.535849761962891
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,48,8,128,1,float16,float16,0,16.21495819091797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,48,1,128,1,float16,float16,0,8.08825912475586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,48,1,128,1,float16,float16,0,4.044825744628906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,48,2,128,1,float16,float16,0,8.198442840576172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,48,4,128,1,float16,float16,0,8.133567810058594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,48,8,128,1,float16,float16,0,8.325011444091796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,48,2,128,1,float16,float16,0,4.093649673461914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,48,4,128,1,float16,float16,0,4.17590560913086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,48,8,128,1,float16,float16,0,4.1964366912841795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,48,1,128,1,float16,float16,0,18.215208435058592
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,48,48,128,1,float16,float16,0,20.110371398925782
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,48,2,128,1,float16,float16,0,18.585354614257813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,48,4,128,1,float16,float16,0,18.450131225585938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,48,1,128,1,float16,float16,0,9.319321441650391
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,48,2,128,1,float16,float16,0,9.125679779052735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,48,48,128,1,float16,float16,0,10.131982421875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,48,2,128,1,float16,float16,0,36.151589965820314
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,48,1,128,1,float16,float16,0,36.66545104980469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,48,4,128,1,float16,float16,0,36.847103881835935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,48,48,128,1,float16,float16,0,5.13940658569336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,48,8,128,1,float16,float16,0,37.08906860351563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,48,8,128,1,float16,float16,0,18.53007354736328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,48,48,128,1,float16,float16,0,2.639748764038086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,48,4,128,1,float16,float16,0,9.404937744140625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,48,1,128,1,float16,float16,0,4.549507141113281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,48,8,128,1,float16,float16,0,9.305652618408203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,48,2,128,1,float16,float16,0,4.620716857910156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,48,4,128,1,float16,float16,0,4.661457443237305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,48,1,128,1,float16,float16,0,2.3267663955688476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,48,8,128,1,float16,float16,0,4.760063934326172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,48,2,128,1,float16,float16,0,2.338515281677246
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,48,4,128,1,float16,float16,0,2.435465621948242
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,48,8,128,1,float16,float16,0,2.412089538574219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,48,1,128,1,float16,float16,0,16.538763427734374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,48,4,128,1,float16,float16,0,16.586460876464844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,48,2,128,1,float16,float16,0,16.83686218261719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,48,48,128,1,float16,float16,0,19.102436828613282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,48,1,128,1,float16,float16,0,8.405912017822265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,48,2,128,1,float16,float16,0,8.405638122558594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,48,48,128,1,float16,float16,0,9.37951202392578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,48,1,128,1,float16,float16,0,33.1058837890625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,48,2,128,1,float16,float16,0,33.29353332519531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,48,48,128,1,float16,float16,0,4.709580612182617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,48,4,128,1,float16,float16,0,33.479916381835935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,48,8,128,1,float16,float16,0,33.512628173828126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,48,8,128,1,float16,float16,0,16.9039794921875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,48,4,128,1,float16,float16,0,8.589888000488282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,48,1,128,1,float16,float16,0,4.120107269287109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,48,48,128,1,float16,float16,0,2.3972816467285156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,48,8,128,1,float16,float16,0,8.543256378173828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,48,2,128,1,float16,float16,0,4.193051147460937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,48,1,128,1,float16,float16,0,2.0920591354370117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,48,4,128,1,float16,float16,0,4.155073547363282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,48,8,128,1,float16,float16,0,4.256111907958984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,48,48,128,1,float16,float16,0,1.2974656105041504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,48,1,128,1,float16,float16,0,1.1137616157531738
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,48,2,128,1,float16,float16,0,1.1229455947875977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,48,2,128,1,float16,float16,0,2.1756752014160154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,48,4,128,1,float16,float16,0,2.146713638305664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,48,8,128,1,float16,float16,0,2.164542388916016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,48,4,128,1,float16,float16,0,1.1326623916625977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,48,8,128,1,float16,float16,0,1.1793184280395508
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,48,1,128,1,float16,float16,0,9.444817352294923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,48,2,128,1,float16,float16,0,9.442902374267579
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,48,4,128,1,float16,float16,0,9.633670043945312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,48,48,128,1,float16,float16,0,11.181916809082031
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,48,1,128,1,float16,float16,0,4.7627918243408205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,48,48,128,1,float16,float16,0,5.552081680297851
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,48,2,128,1,float16,float16,0,4.751651382446289
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,48,2,128,1,float16,float16,0,19.036402893066406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,48,1,128,1,float16,float16,0,19.229293823242188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,48,4,128,1,float16,float16,0,19.152259826660156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,48,8,128,1,float16,float16,0,19.65284423828125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,48,8,128,1,float16,float16,0,9.687051391601562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,48,48,128,1,float16,float16,0,2.8223072052001954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,48,4,128,1,float16,float16,0,4.812176132202149
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,48,8,128,1,float16,float16,0,4.794251251220703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,48,48,128,1,float16,float16,0,1.4556528091430665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,48,2,128,1,float16,float16,0,2.387691116333008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,48,1,128,1,float16,float16,0,2.3532543182373047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,48,1,128,1,float16,float16,0,1.2386879920959473
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,48,2,128,1,float16,float16,0,1.2259424209594727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,48,4,128,1,float16,float16,0,2.4675983428955077
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,48,8,128,1,float16,float16,0,2.435838317871094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,48,48,128,1,float16,float16,0,0.7918000221252441
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,48,1,128,1,float16,float16,0,0.6437215805053711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,48,2,128,1,float16,float16,0,0.6595583915710449
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,48,4,128,1,float16,float16,0,1.244649600982666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,48,8,128,1,float16,float16,0,1.2840463638305664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,48,4,128,1,float16,float16,0,0.6847040176391601
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,48,8,128,1,float16,float16,0,0.6909264087677002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,48,1,128,1,float16,float16,0,8.846059417724609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,48,48,128,1,float16,float16,0,11.052967834472657
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,48,2,128,1,float16,float16,0,8.741117095947265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,48,48,128,1,float16,float16,0,5.535417556762695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,48,4,128,1,float16,float16,0,8.8404541015625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,48,1,128,1,float16,float16,0,17.482034301757814
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,48,2,128,1,float16,float16,0,17.51326141357422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,48,4,128,1,float16,float16,0,17.566297912597655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,48,1,128,1,float16,float16,0,4.2967376708984375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,48,8,128,1,float16,float16,0,17.99596252441406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,48,8,128,1,float16,float16,0,9.150041961669922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,48,1,128,1,float16,float16,0,2.1207391738891603
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,48,2,128,1,float16,float16,0,2.184940719604492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,48,4,128,1,float16,float16,0,2.209329605102539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,48,48,128,1,float16,float16,0,2.7383663177490236
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,48,8,128,1,float16,float16,0,2.297102355957031
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,48,1,128,1,float16,float16,0,1.1042271614074708
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,48,48,128,1,float16,float16,0,1.3899392127990722
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,48,2,128,1,float16,float16,0,1.1250543594360352
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,48,4,128,1,float16,float16,0,4.416574478149414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,48,2,128,1,float16,float16,0,4.381558227539062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,48,4,128,1,float16,float16,0,1.1499759674072265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,48,8,128,1,float16,float16,0,4.503054428100586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,48,48,128,1,float16,float16,0,0.7406832218170166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,48,1,128,1,float16,float16,0,0.5776912212371826
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,48,2,128,1,float16,float16,0,0.6008304119110107
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,48,1,128,1,float16,float16,0,0.3414175987243652
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,48,2,128,1,float16,float16,0,0.3316351890563965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,48,4,128,1,float16,float16,0,0.6347824096679687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,48,8,128,1,float16,float16,0,0.6358943939208984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,48,4,128,1,float16,float16,0,0.34191360473632815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,48,8,128,1,float16,float16,0,0.3519344091415405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,48,8,128,1,float16,float16,0,1.20065279006958
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,48,48,128,1,float16,float16,0,0.41850080490112307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,48,1,128,1,float16,float16,0,5.026467132568359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,48,48,128,1,float16,float16,0,6.694385528564453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,48,4,128,1,float16,float16,0,5.099252700805664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,48,2,128,1,float16,float16,0,5.180806350708008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,48,2,128,1,float16,float16,0,10.207323455810547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,48,4,128,1,float16,float16,0,10.358505249023438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,48,8,128,1,float16,float16,0,10.651473236083984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,48,8,128,1,float16,float16,0,5.277398300170899
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,48,1,128,1,float16,float16,0,2.5341344833374024
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,48,2,128,1,float16,float16,0,2.5854560852050783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,48,48,128,1,float16,float16,0,3.3207584381103517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,48,1,128,1,float16,float16,0,1.2789471626281739
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,48,48,128,1,float16,float16,0,1.72979679107666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,48,2,128,1,float16,float16,0,1.296776008605957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,48,4,128,1,float16,float16,0,1.3219759941101075
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,48,8,128,1,float16,float16,0,1.3292367935180665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,48,4,128,1,float16,float16,0,2.5862096786499023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,48,8,128,1,float16,float16,0,2.6346288681030274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,48,48,128,1,float16,float16,0,0.8705056190490723
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,48,1,128,1,float16,float16,0,0.6763216018676758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,48,48,128,1,float16,float16,0,0.4870704174041748
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,48,4,128,1,float16,float16,0,0.6857359886169434
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,48,1,128,1,float16,float16,0,10.182835388183594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,48,8,128,1,float16,float16,0,0.6930416107177735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,48,1,128,1,float16,float16,0,0.3796080112457275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,48,2,128,1,float16,float16,0,0.37047040462493896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,48,4,128,1,float16,float16,0,0.3829296112060547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,48,8,128,1,float16,float16,0,0.3855423927307129
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,48,1,128,1,float16,float16,0,0.2019439935684204
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,48,2,128,1,float16,float16,0,0.20314559936523438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,48,4,128,1,float16,float16,0,0.22202720642089843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,48,2,128,1,float16,float16,0,0.6849855899810791
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,48,48,128,1,float16,float16,0,0.26698238849639894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,48,8,128,1,float16,float16,0,0.22333760261535646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,48,48,128,1,float16,float16,0,6.9870147705078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,48,1,128,1,float16,float16,0,4.792027282714844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,48,2,128,1,float16,float16,0,4.8993408203125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,48,4,128,1,float16,float16,0,4.869019317626953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,48,2,128,1,float16,float16,0,9.688646697998047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,48,1,128,1,float16,float16,0,9.741849517822265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,48,4,128,1,float16,float16,0,9.957798767089844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,48,8,128,1,float16,float16,0,10.128545379638672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,48,1,128,1,float16,float16,0,2.417531204223633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,48,2,128,1,float16,float16,0,2.4763904571533204
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,48,48,128,1,float16,float16,0,3.517193603515625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,48,1,128,1,float16,float16,0,1.191483211517334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,48,4,128,1,float16,float16,0,2.4412607192993163
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,48,8,128,1,float16,float16,0,2.536292839050293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,48,2,128,1,float16,float16,0,1.2409008026123047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,48,8,128,1,float16,float16,0,5.0845073699951175
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,48,4,128,1,float16,float16,0,1.2515151977539063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,48,8,128,1,float16,float16,0,1.2715408325195312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,48,48,128,1,float16,float16,0,0.890447998046875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,48,1,128,1,float16,float16,0,0.6387631893157959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,48,2,128,1,float16,float16,0,0.6344687938690186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,48,4,128,1,float16,float16,0,0.6634560108184815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,48,48,128,1,float16,float16,0,0.4790927886962891
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,48,2,128,1,float16,float16,0,0.3471375942230225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,48,8,128,1,float16,float16,0,0.6596560001373291
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,48,4,128,1,float16,float16,0,0.34444639682769773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,48,1,128,1,float16,float16,0,0.18448159694671631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,48,48,128,1,float16,float16,0,0.2657552003860474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,48,8,128,1,float16,float16,0,0.35852000713348386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,48,2,128,1,float16,float16,0,0.19418400526046753
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,48,4,128,1,float16,float16,0,0.19683200120925903
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,48,1,128,1,float16,float16,0,0.11355520486831665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,48,48,128,1,float16,float16,0,0.15414719581604003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,48,2,128,1,float16,float16,0,0.12047200202941895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,48,8,128,1,float16,float16,0,0.20403358936309815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,48,8,128,1,float16,float16,0,0.11842080354690551
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,48,48,128,1,float16,float16,0,1.7701583862304688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,48,1,128,1,float16,float16,0,0.32559359073638916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,48,4,128,1,float16,float16,0,0.1172976016998291
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,48,1,128,1,float16,float16,0,2.7490400314331054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,48,1,128,1,float16,float16,0,5.711902236938476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,48,2,128,1,float16,float16,0,5.782115173339844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,48,4,128,1,float16,float16,0,6.018656158447266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,48,48,128,1,float16,float16,0,5.057966232299805
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,48,2,128,1,float16,float16,0,2.874430465698242
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,48,8,128,1,float16,float16,0,6.268073654174804
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,48,1,128,1,float16,float16,0,1.4052543640136719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,48,2,128,1,float16,float16,0,1.429916763305664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,48,4,128,1,float16,float16,0,1.512284755706787
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,48,8,128,1,float16,float16,0,1.5481632232666016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,48,1,128,1,float16,float16,0,0.7417344093322754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,48,48,128,1,float16,float16,0,2.5405887603759765
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,48,4,128,1,float16,float16,0,2.9323040008544923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,48,8,128,1,float16,float16,0,3.1090015411376952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,48,48,128,1,float16,float16,0,1.2687583923339845
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,48,8,128,1,float16,float16,0,0.7876175880432129
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,48,1,128,1,float16,float16,0,0.372216010093689
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,48,48,128,1,float16,float16,0,0.6647840023040772
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,48,4,128,1,float16,float16,0,0.39645919799804685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,48,48,128,1,float16,float16,0,0.3533695936203003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,48,8,128,1,float16,float16,0,0.40328159332275393
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,48,1,128,1,float16,float16,0,0.2008336067199707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,48,2,128,1,float16,float16,0,0.20559520721435548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,48,4,128,1,float16,float16,0,0.21448640823364257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,48,8,128,1,float16,float16,0,0.2210767984390259
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,48,1,128,1,float16,float16,0,0.11720000505447388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,48,2,128,1,float16,float16,0,0.12002079486846924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,48,4,128,1,float16,float16,0,0.12239999771118164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,48,8,128,1,float16,float16,0,0.12626080513000487
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,48,48,128,1,float16,float16,0,0.10859999656677247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,48,1,128,1,float16,float16,0,0.07364159822463989
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,48,2,128,1,float16,float16,0,0.07048640251159669
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,48,4,128,1,float16,float16,0,0.07804800271987915
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,48,8,128,1,float16,float16,0,0.07963680028915406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,48,48,128,1,float16,float16,0,0.06742240190505981
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,48,1,128,1,float16,float16,0,0.04180159866809845
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,48,2,128,1,float16,float16,0,0.04690560102462769
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,48,2,128,1,float16,float16,0,0.7419456005096435
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,48,4,128,1,float16,float16,0,0.7698192119598388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,48,2,128,1,float16,float16,0,0.39358720779418943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,48,4,128,1,float16,float16,0,0.04472320079803467
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,48,48,128,1,float16,float16,0,0.1957792043685913
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,48,8,128,1,float16,float16,0,0.04182080030441284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,48,1,128,1,float16,float16,0,0.9175456047058106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,48,2,128,1,float16,float16,0,0.9325311660766602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,48,4,128,1,float16,float16,0,0.9710111618041992
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,48,1,128,1,float16,float16,0,1.8302335739135742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,48,4,128,1,float16,float16,0,1.9178159713745118
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,48,1,128,1,float16,float16,0,0.47032642364501953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,48,8,128,1,float16,float16,0,2.0778751373291016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,48,48,128,1,float16,float16,0,2.1378496170043944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,48,8,128,1,float16,float16,0,1.0624879837036132
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,48,48,128,1,float16,float16,0,1.0937520027160645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,48,2,128,1,float16,float16,0,1.8544271469116211
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,48,2,128,1,float16,float16,0,0.2528000116348267
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,48,48,128,1,float16,float16,0,0.571619176864624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,48,1,128,1,float16,float16,0,0.24982719421386718
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,48,2,128,1,float16,float16,0,0.48645758628845215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,48,8,128,1,float16,float16,0,0.2848623991012573
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,48,4,128,1,float16,float16,0,0.5026192188262939
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,48,4,128,1,float16,float16,0,0.143779194355011
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,48,48,128,1,float16,float16,0,0.29662559032440183
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,48,48,128,1,float16,float16,0,0.165011203289032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,48,1,128,1,float16,float16,0,0.1371440052986145
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,48,4,128,1,float16,float16,0,0.0840287983417511
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,48,2,128,1,float16,float16,0,0.08097119927406311
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,48,8,128,1,float16,float16,0,0.08846399784088135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,48,48,128,1,float16,float16,0,0.08840799927711487
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,48,1,128,1,float16,float16,0,0.04828479886054993
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,48,2,128,1,float16,float16,0,0.04984480142593384
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,48,4,128,1,float16,float16,0,0.051830399036407473
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,48,8,128,1,float16,float16,0,0.05548959970474243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,48,48,128,1,float16,float16,0,0.049963200092315675
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,48,2,128,1,float16,float16,0,0.030100798606872557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,48,1,128,1,float16,float16,0,0.031886398792266846
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,48,4,128,1,float16,float16,0,0.028460800647735596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,48,8,128,1,float16,float16,0,0.032329601049423215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,48,48,128,1,float16,float16,0,0.02755360007286072
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,48,1,128,1,float16,float16,0,0.02339999973773956
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,48,2,128,1,float16,float16,0,0.02303680032491684
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,48,4,128,1,float16,float16,0,0.02374880015850067
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,48,8,128,1,float16,float16,0,0.023475199937820435
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,48,1,128,1,float16,float16,0,0.703542423248291
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,48,8,128,1,float16,float16,0,0.5381872177124023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,48,2,128,1,float16,float16,0,0.7231264114379883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,48,2,128,1,float16,float16,0,0.13897119760513305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,48,8,128,1,float16,float16,0,0.1532431960105896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,48,4,128,1,float16,float16,0,0.7824639797210693
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,48,8,128,1,float16,float16,0,0.8769807815551758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,48,1,128,1,float16,float16,0,0.08091359734535217
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,48,1,128,1,float16,float16,0,0.36173760890960693
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,48,48,128,1,float16,float16,0,1.0565872192382812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,48,2,128,1,float16,float16,0,0.3747231960296631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,48,4,128,1,float16,float16,0,0.40690078735351565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,48,1,128,1,float16,float16,0,0.19200639724731444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,48,2,128,1,float16,float16,0,0.19974559545516968
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,48,8,128,1,float16,float16,0,0.45658559799194337
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,48,4,128,1,float16,float16,0,0.21245760917663575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,48,8,128,1,float16,float16,0,0.23845279216766357
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,48,1,128,1,float16,float16,0,0.10697439908981324
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,48,48,128,1,float16,float16,0,0.28551039695739744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,48,8,128,1,float16,float16,0,0.1311408042907715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,48,1,128,1,float16,float16,0,0.06466879844665527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,48,2,128,1,float16,float16,0,0.06630240082740783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,48,48,128,1,float16,float16,0,0.15940320491790771
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,48,4,128,1,float16,float16,0,0.06857600212097167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,48,8,128,1,float16,float16,0,0.0748416006565094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,48,48,128,1,float16,float16,0,0.08339999914169312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,48,2,128,1,float16,float16,0,0.03867680132389069
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,48,4,128,1,float16,float16,0,0.04010559916496277
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,48,8,128,1,float16,float16,0,0.0443583995103836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,48,48,128,1,float16,float16,0,0.04235360026359558
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,48,1,128,1,float16,float16,0,0.023907199501991272
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,48,2,128,1,float16,float16,0,0.021940800547599792
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,48,4,128,1,float16,float16,0,0.02399359941482544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,48,4,128,1,float16,float16,0,0.2626944065093994
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,48,8,128,1,float16,float16,0,0.023992000520229338
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,48,48,128,1,float16,float16,0,0.023060800135135652
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,48,2,128,1,float16,float16,0,0.01863359957933426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,48,1,128,1,float16,float16,0,0.01828159987926483
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,48,4,128,1,float16,float16,0,0.019079999625682832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,48,8,128,1,float16,float16,0,0.01900320053100586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,48,48,128,1,float16,float16,0,0.020377600193023683
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,48,1,128,1,float16,float16,0,0.017980800569057466
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,48,2,128,1,float16,float16,0,0.017878399789333345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,48,4,128,1,float16,float16,0,0.017820799350738527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,48,8,128,1,float16,float16,0,0.01823360025882721
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,48,4,128,1,float16,float16,0,0.4022416114807129
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,48,1,128,1,float16,float16,0,0.3524224042892456
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,48,2,128,1,float16,float16,0,0.36588799953460693
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,48,1,128,1,float16,float16,0,0.1905071973800659
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,48,8,128,1,float16,float16,0,0.4573488235473633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,48,48,128,1,float16,float16,0,0.5432464122772217
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,48,2,128,1,float16,float16,0,0.19407520294189454
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,48,4,128,1,float16,float16,0,0.21163840293884278
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,48,4,128,1,float16,float16,0,0.11554239988327027
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,48,8,128,1,float16,float16,0,0.24414560794830323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,48,48,128,1,float16,float16,0,0.28305759429931643
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,48,1,128,1,float16,float16,0,0.10287679433822632
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,48,2,128,1,float16,float16,0,0.10714080333709716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,48,4,128,1,float16,float16,0,0.11729600429534912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,48,1,128,1,float16,float16,0,0.06128799915313721
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,48,8,128,1,float16,float16,0,0.13338079452514648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,48,2,128,1,float16,float16,0,0.06308959722518921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,48,1,128,1,float16,float16,0,0.036432000994682315
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,48,4,128,1,float16,float16,0,0.065830397605896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,48,8,128,1,float16,float16,0,0.0741216003894806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,48,1,128,1,float16,float16,0,0.03152639865875244
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,48,48,128,1,float16,float16,0,0.08401600122451783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,48,2,128,1,float16,float16,0,0.032769599556922914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,48,4,128,1,float16,float16,0,0.03553920090198517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,48,8,128,1,float16,float16,0,0.04070079922676086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,48,48,128,1,float16,float16,0,0.04416320025920868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,48,1,128,1,float16,float16,0,0.020342400670051573
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,48,2,128,1,float16,float16,0,0.020291200280189513
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,48,4,128,1,float16,float16,0,0.020268799364566804
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,48,8,128,1,float16,float16,0,0.0204927995800972
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,48,48,128,1,float16,float16,0,0.021414400637149812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,48,1,128,1,float16,float16,0,0.01664000004529953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,48,2,128,1,float16,float16,0,0.016620799899101257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,48,4,128,1,float16,float16,0,0.016521599888801575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,48,8,128,1,float16,float16,0,0.016804799437522888
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,48,48,128,1,float16,float16,0,0.017560000717639922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,48,1,128,1,float16,float16,0,0.015297600626945495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,48,2,128,1,float16,float16,0,0.01528639942407608
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,48,4,128,1,float16,float16,0,0.015335999429225922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,48,8,128,1,float16,float16,0,0.01573919951915741
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,48,48,128,1,float16,float16,0,0.016212800145149232
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,48,1,128,1,float16,float16,0,0.015097600221633912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,48,2,128,1,float16,float16,0,0.015001599490642548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,48,8,128,1,float16,float16,0,0.015279999375343323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,48,1,128,1,float16,float16,0,0.17961599826812744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,48,2,128,1,float16,float16,0,0.1873952031135559
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,48,4,128,1,float16,float16,0,0.20761120319366455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,48,8,128,1,float16,float16,0,0.25228960514068605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,48,1,128,1,float16,float16,0,0.10045440196990967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,48,2,128,1,float16,float16,0,0.1068608045578003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,48,48,128,1,float16,float16,0,0.3610480070114136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,48,4,128,1,float16,float16,0,0.1118127942085266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,48,48,128,1,float16,float16,0,0.5420464038848877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,48,48,128,1,float16,float16,0,0.18765759468078613
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,48,1,128,1,float16,float16,0,0.06205120086669922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,48,2,128,1,float16,float16,0,0.06407679915428162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,48,8,128,1,float16,float16,0,0.07859359979629517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,48,2,128,1,float16,float16,0,0.10963040590286255
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,48,4,128,1,float16,float16,0,0.06669600009918213
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,48,1,128,1,float16,float16,0,0.032118400931358336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,48,48,128,1,float16,float16,0,0.10120960474014282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,48,4,128,1,float16,float16,0,0.03559359908103943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,48,48,128,1,float16,float16,0,0.15445120334625245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,48,2,128,1,float16,float16,0,0.03266560137271881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,48,2,128,1,float16,float16,0,0.019780799746513367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,48,8,128,1,float16,float16,0,0.04693439900875092
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,48,48,128,1,float16,float16,0,0.056443202495574954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,48,4,128,1,float16,float16,0,0.015078400075435639
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,48,8,128,1,float16,float16,0,0.025407999753952026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,48,48,128,1,float16,float16,0,0.02866080105304718
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,48,2,128,1,float16,float16,0,0.01619199961423874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,48,4,128,1,float16,float16,0,0.01648160070180893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,48,8,128,1,float16,float16,0,0.018622399866580965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,48,48,128,1,float16,float16,0,0.01902880072593689
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,48,1,128,1,float16,float16,0,0.01515199989080429
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,48,2,128,1,float16,float16,0,0.015220800042152404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,48,4,128,1,float16,float16,0,0.015275199711322785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,48,48,128,1,float16,float16,0,0.01650400012731552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,48,1,128,1,float16,float16,0,0.015033599734306336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,48,4,128,1,float16,float16,0,0.015014399588108063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,48,2,128,1,float16,float16,0,0.015015999972820281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,48,8,128,1,float16,float16,0,0.015199999511241912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,48,1,128,1,float16,float16,0,0.014800000190734863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,48,2,128,1,float16,float16,0,0.014769600331783294
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,48,4,128,1,float16,float16,0,0.014924800395965577
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,48,8,128,1,float16,float16,0,0.13815200328826904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,48,8,128,1,float16,float16,0,0.015087999403476715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,48,1,128,1,float16,float16,0,0.10110080242156982
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,48,2,128,1,float16,float16,0,0.10349760055541993
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,48,4,128,1,float16,float16,0,0.12528480291366578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,48,8,128,1,float16,float16,0,0.13449440002441407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,48,1,128,1,float16,float16,0,0.06245599985122681
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,48,48,128,1,float16,float16,0,0.28951520919799806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,48,2,128,1,float16,float16,0,0.06233919858932495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,48,4,128,1,float16,float16,0,0.07298240065574646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,48,8,128,1,float16,float16,0,0.07701759934425353
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,48,1,128,1,float16,float16,0,0.03217119872570038
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,48,48,128,1,float16,float16,0,0.15142560005187988
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,48,2,128,1,float16,float16,0,0.033404800295829776
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,48,4,128,1,float16,float16,0,0.042454400658607484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,48,8,128,1,float16,float16,0,0.046177598834037784
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,48,1,128,1,float16,float16,0,0.02041600048542023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,48,48,128,1,float16,float16,0,0.08309599757194519
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,48,2,128,1,float16,float16,0,0.02029760032892227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,48,4,128,1,float16,float16,0,0.01974720060825348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,48,4,128,1,float16,float16,0,0.02593280076980591
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,48,8,128,1,float16,float16,0,0.026155200600624085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,48,48,128,1,float16,float16,0,0.04300960004329681
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,48,1,128,1,float16,float16,0,0.016497600078582763
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,48,1,128,1,float16,float16,0,0.016209599375724793
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,48,2,128,1,float16,float16,0,0.016172799468040466
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,48,4,128,1,float16,float16,0,0.018192000687122345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,48,8,128,1,float16,float16,0,0.01804800033569336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,48,48,128,1,float16,float16,0,0.026510399580001832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,48,1,128,1,float16,float16,0,0.014929600059986115
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,48,2,128,1,float16,float16,0,0.015147200226783753
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,48,4,128,1,float16,float16,0,0.01528480052947998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,48,8,128,1,float16,float16,0,0.015160000324249268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,48,48,128,1,float16,float16,0,0.01828960031270981
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,48,1,128,1,float16,float16,0,0.014923200011253357
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,48,2,128,1,float16,float16,0,0.015014399588108063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,48,4,128,1,float16,float16,0,0.014971199631690978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,48,1,128,1,float16,float16,0,0.014796799421310425
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,48,2,128,1,float16,float16,0,0.014913600683212281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,48,48,128,1,float16,float16,0,0.016153599321842193
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,48,4,128,1,float16,float16,0,0.014847999811172486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,48,8,128,1,float16,float16,0,0.014587199687957764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,48,48,128,1,float16,float16,0,0.015887999534606935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,48,48,128,1,float16,float16,0,0.01568640023469925
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,48,1,128,1,float16,float16,0,0.014612799882888794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,48,4,128,1,float16,float16,0,0.01467519998550415
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,48,8,128,1,float16,float16,0,0.014732800424098969
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,48,1,128,1,float16,float16,0,0.027036800980567932
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,48,2,128,1,float16,float16,0,0.034299200773239134
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,48,4,128,1,float16,float16,0,0.05179679989814758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,48,1,128,1,float16,float16,0,0.018743999302387238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,48,8,128,1,float16,float16,0,0.0892080008983612
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,48,2,128,1,float16,float16,0,0.024804799258708952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,48,48,128,1,float16,float16,0,0.22482399940490722
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,48,4,128,1,float16,float16,0,0.032153600454330446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,48,8,128,1,float16,float16,0,0.050032001733779904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,48,48,128,1,float16,float16,0,0.11932320594787597
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,48,1,128,1,float16,float16,0,0.015862399339675905
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,48,2,128,1,float16,float16,0,0.017580799758434296
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,48,4,128,1,float16,float16,0,0.023473599553108217
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,48,8,128,1,float16,float16,0,0.031067198514938353
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,48,1,128,1,float16,float16,0,0.014875200390815736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,48,48,128,1,float16,float16,0,0.06696799993515015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,48,2,128,1,float16,float16,0,0.015041600167751312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,48,4,128,1,float16,float16,0,0.016841599345207216
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,48,8,128,1,float16,float16,0,0.02272319942712784
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,48,48,128,1,float16,float16,0,0.03977920114994049
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,48,1,128,1,float16,float16,0,0.014316800236701965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,48,2,128,1,float16,float16,0,0.014351999759674073
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,48,4,128,1,float16,float16,0,0.014368000626564025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,48,8,128,1,float16,float16,0,0.016297599673271178
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,48,1,128,1,float16,float16,0,0.014201599359512328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,48,2,128,1,float16,float16,0,0.014183999598026275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,48,4,128,1,float16,float16,0,0.014312000572681427
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,48,8,128,1,float16,float16,0,0.014395199716091156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,48,48,128,1,float16,float16,0,0.01751679927110672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,48,1,128,1,float16,float16,0,0.013910399377346038
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,48,4,128,1,float16,float16,0,0.014254400134086609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,48,8,128,1,float16,float16,0,0.014161600172519684
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,48,48,128,1,float16,float16,0,0.01589120030403137
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,48,1,128,1,float16,float16,0,0.01401440054178238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,48,2,128,1,float16,float16,0,0.013825599849224091
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,48,4,128,1,float16,float16,0,0.013940800726413728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,48,8,128,1,float16,float16,0,0.013911999762058258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,48,48,128,1,float16,float16,0,0.01542080044746399
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,48,1,128,1,float16,float16,0,0.020179200172424316
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,48,1,128,1,float16,float16,0,0.013780799508094788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,48,4,128,1,float16,float16,0,0.013753600418567657
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,48,2,128,1,float16,float16,0,0.014169600605964661
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,48,8,128,1,float16,float16,0,0.013790400326251983
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,48,8,128,1,float16,float16,0,0.015692800283432007
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,48,2,128,1,float16,float16,0,0.014846399426460266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,48,48,128,1,float16,float16,0,0.025276800990104674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,48,2,128,1,float16,float16,0,0.014099200069904328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,48,8,128,1,float16,float16,0,0.014876799285411834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,40,1,128,1,float16,float16,0,53.578118896484376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,40,2,128,1,float16,float16,0,52.66104736328125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,40,4,128,1,float16,float16,0,52.9021240234375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,40,1,128,1,float16,float16,0,26.541830444335936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,40,1,128,1,float16,float16,0,104.12227783203124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,40,2,128,1,float16,float16,0,105.85928955078126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,40,4,128,1,float16,float16,0,105.8450927734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,40,8,128,1,float16,float16,0,106.39610595703125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,40,8,128,1,float16,float16,0,54.1913818359375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,40,2,128,1,float16,float16,0,26.44771728515625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,40,1,128,1,float16,float16,0,13.385691833496093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,40,2,128,1,float16,float16,0,13.567027282714843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,40,4,128,1,float16,float16,0,13.464089965820312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,40,4,128,1,float16,float16,0,26.396173095703126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,40,8,128,1,float16,float16,0,26.994351196289063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,40,8,128,1,float16,float16,0,13.570414733886718
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,40,1,128,1,float16,float16,0,60.640472412109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,40,2,128,1,float16,float16,0,59.56337890625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,40,4,128,1,float16,float16,0,59.36641235351563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,40,8,128,1,float16,float16,0,60.61417236328125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,40,1,128,1,float16,float16,0,30.24619140625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,40,2,128,1,float16,float16,0,29.876058959960936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,40,4,128,1,float16,float16,0,29.784661865234376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,40,8,128,1,float16,float16,0,30.403610229492188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,40,1,128,1,float16,float16,0,15.06917724609375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,40,2,128,1,float16,float16,0,14.967247009277344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,40,4,128,1,float16,float16,0,15.24598388671875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,40,1,128,1,float16,float16,0,7.62066421508789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,40,8,128,1,float16,float16,0,15.231654357910156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,40,2,128,1,float16,float16,0,7.610380554199219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,40,4,128,1,float16,float16,0,7.5853523254394535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,40,40,128,1,float16,float16,0,8.039174652099609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,40,8,128,1,float16,float16,0,7.7430061340332035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,40,40,128,1,float16,float16,0,14.123880004882812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,40,40,128,1,float16,float16,0,16.23384246826172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,40,40,128,1,float16,float16,0,27.6662109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,40,40,128,1,float16,float16,0,31.640280151367186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,40,40,128,1,float16,float16,0,22.67981414794922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,40,1,128,1,float16,float16,0,41.55775146484375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,40,1,128,1,float16,float16,0,20.491557312011718
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,40,2,128,1,float16,float16,0,41.0916748046875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,40,2,128,1,float16,float16,0,20.753892517089845
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,40,4,128,1,float16,float16,0,42.08925170898438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,40,40,128,1,float16,float16,0,55.86275634765625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,40,8,128,1,float16,float16,0,42.175778198242185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,40,1,128,1,float16,float16,0,10.346752166748047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,40,40,128,1,float16,float16,0,5.74987678527832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,40,2,128,1,float16,float16,0,10.490325164794921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,40,4,128,1,float16,float16,0,10.657729339599609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,40,40,128,1,float16,float16,0,11.146324920654298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,40,8,128,1,float16,float16,0,10.724161529541016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,40,4,128,1,float16,float16,0,21.372395324707032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,40,8,128,1,float16,float16,0,21.128384399414063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,40,1,128,1,float16,float16,0,5.269804763793945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,40,2,128,1,float16,float16,0,5.32598876953125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,40,4,128,1,float16,float16,0,5.33862075805664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,40,8,128,1,float16,float16,0,5.452857589721679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,40,1,128,1,float16,float16,0,26.90361328125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,40,2,128,1,float16,float16,0,26.872763061523436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,40,40,128,1,float16,float16,0,28.647113037109374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,40,4,128,1,float16,float16,0,27.111294555664063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,40,40,128,1,float16,float16,0,14.591799926757812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,40,1,128,1,float16,float16,0,13.263252258300781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,40,2,128,1,float16,float16,0,13.57901611328125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,40,1,128,1,float16,float16,0,53.176544189453125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,40,2,128,1,float16,float16,0,54.05418090820312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,40,40,128,1,float16,float16,0,7.414409637451172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,40,8,128,1,float16,float16,0,54.33128051757812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,40,4,128,1,float16,float16,0,54.686529541015624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,40,8,128,1,float16,float16,0,27.205459594726562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,40,4,128,1,float16,float16,0,13.494490051269532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,40,8,128,1,float16,float16,0,13.542031860351562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,40,1,128,1,float16,float16,0,6.740144348144531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,40,2,128,1,float16,float16,0,6.775920104980469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,40,40,128,1,float16,float16,0,3.755656051635742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,40,1,128,1,float16,float16,0,3.4419185638427736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,40,2,128,1,float16,float16,0,3.395598220825195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,40,4,128,1,float16,float16,0,3.443348693847656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,40,4,128,1,float16,float16,0,6.858930969238282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,40,8,128,1,float16,float16,0,7.035708618164063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,40,8,128,1,float16,float16,0,3.5983551025390623
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,40,1,128,1,float16,float16,0,15.153758239746093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,40,2,128,1,float16,float16,0,15.545021057128906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,40,4,128,1,float16,float16,0,15.474940490722656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,40,40,128,1,float16,float16,0,16.734544372558595
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,40,1,128,1,float16,float16,0,7.6740272521972654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,40,40,128,1,float16,float16,0,8.576884460449218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,40,2,128,1,float16,float16,0,7.613502502441406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,40,1,128,1,float16,float16,0,30.347476196289062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,40,2,128,1,float16,float16,0,30.591519165039063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,40,4,128,1,float16,float16,0,30.487759399414063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,40,8,128,1,float16,float16,0,31.032308959960936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,40,8,128,1,float16,float16,0,15.56042938232422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,40,4,128,1,float16,float16,0,7.7985069274902346
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,40,1,128,1,float16,float16,0,3.841409683227539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,40,8,128,1,float16,float16,0,7.980854034423828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,40,40,128,1,float16,float16,0,4.187318420410156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,40,2,128,1,float16,float16,0,3.8585536956787108
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,40,40,128,1,float16,float16,0,2.1749088287353517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,40,4,128,1,float16,float16,0,3.8542606353759767
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,40,8,128,1,float16,float16,0,4.233832168579101
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,40,1,128,1,float16,float16,0,2.004572868347168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,40,2,128,1,float16,float16,0,2.0062416076660154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,40,4,128,1,float16,float16,0,2.068806457519531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,40,8,128,1,float16,float16,0,2.038737678527832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,40,1,128,1,float16,float16,0,13.784233093261719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,40,4,128,1,float16,float16,0,13.873663330078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,40,2,128,1,float16,float16,0,14.05316162109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,40,40,128,1,float16,float16,0,15.813568115234375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,40,1,128,1,float16,float16,0,7.030404663085937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,40,40,128,1,float16,float16,0,7.851163482666015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,40,2,128,1,float16,float16,0,6.975361633300781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,40,1,128,1,float16,float16,0,27.549853515625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,40,2,128,1,float16,float16,0,27.640353393554687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,40,4,128,1,float16,float16,0,27.61868896484375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,40,40,128,1,float16,float16,0,4.005670547485352
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,40,8,128,1,float16,float16,0,14.178785705566407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,40,8,128,1,float16,float16,0,28.272161865234374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,40,4,128,1,float16,float16,0,7.127047729492188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,40,1,128,1,float16,float16,0,3.515691375732422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,40,8,128,1,float16,float16,0,7.067475128173828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,40,2,128,1,float16,float16,0,3.457539367675781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,40,40,128,1,float16,float16,0,2.022228813171387
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,40,4,128,1,float16,float16,0,3.493048095703125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,40,1,128,1,float16,float16,0,1.765465545654297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,40,2,128,1,float16,float16,0,1.8281520843505858
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,40,1,128,1,float16,float16,0,0.9174768447875976
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,40,40,128,1,float16,float16,0,1.088047981262207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,40,2,128,1,float16,float16,0,0.9493071556091308
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,40,4,128,1,float16,float16,0,0.9889887809753418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,40,8,128,1,float16,float16,0,3.585171127319336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,40,4,128,1,float16,float16,0,1.8168319702148437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,40,8,128,1,float16,float16,0,1.8321535110473632
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,40,8,128,1,float16,float16,0,0.9943920135498047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,40,1,128,1,float16,float16,0,7.959185791015625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,40,2,128,1,float16,float16,0,7.923671722412109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,40,4,128,1,float16,float16,0,8.157809448242187
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,40,40,128,1,float16,float16,0,9.388008117675781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,40,1,128,1,float16,float16,0,4.028104019165039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,40,40,128,1,float16,float16,0,4.622103881835938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,40,2,128,1,float16,float16,0,4.012041473388672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,40,1,128,1,float16,float16,0,15.778021240234375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,40,2,128,1,float16,float16,0,15.877450561523437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,40,4,128,1,float16,float16,0,15.911761474609374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,40,8,128,1,float16,float16,0,8.1847900390625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,40,8,128,1,float16,float16,0,16.491256713867188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,40,40,128,1,float16,float16,0,2.3735376358032227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,40,4,128,1,float16,float16,0,4.106111907958985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,40,1,128,1,float16,float16,0,2.008576011657715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,40,8,128,1,float16,float16,0,4.086097717285156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,40,1,128,1,float16,float16,0,1.043950366973877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,40,40,128,1,float16,float16,0,1.2408687591552734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,40,2,128,1,float16,float16,0,2.0236751556396486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,40,4,128,1,float16,float16,0,2.0357168197631834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,40,2,128,1,float16,float16,0,1.0855183601379395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,40,8,128,1,float16,float16,0,2.076140785217285
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,40,2,128,1,float16,float16,0,0.5734864234924316
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,40,1,128,1,float16,float16,0,0.5508368015289307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,40,4,128,1,float16,float16,0,1.0827296257019043
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,40,4,128,1,float16,float16,0,0.5934000015258789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,40,8,128,1,float16,float16,0,1.1071136474609375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,40,8,128,1,float16,float16,0,0.6120336055755615
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,40,40,128,1,float16,float16,0,0.6702191829681396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,40,1,128,1,float16,float16,0,7.189300537109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,40,2,128,1,float16,float16,0,7.450498962402344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,40,40,128,1,float16,float16,0,9.002123260498047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,40,4,128,1,float16,float16,0,7.461510467529297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,40,40,128,1,float16,float16,0,4.639833450317383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,40,1,128,1,float16,float16,0,14.580035400390624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,40,2,128,1,float16,float16,0,14.655079650878907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,40,4,128,1,float16,float16,0,14.93970184326172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,40,1,128,1,float16,float16,0,3.6616302490234376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,40,8,128,1,float16,float16,0,7.577593231201172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,40,8,128,1,float16,float16,0,15.245481872558594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,40,2,128,1,float16,float16,0,3.731024169921875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,40,1,128,1,float16,float16,0,1.8266399383544922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,40,2,128,1,float16,float16,0,1.8333776473999024
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,40,4,128,1,float16,float16,0,1.8765392303466797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,40,4,128,1,float16,float16,0,3.708486557006836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,40,8,128,1,float16,float16,0,3.794630432128906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,40,40,128,1,float16,float16,0,1.1995087623596192
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,40,1,128,1,float16,float16,0,0.9322527885437012
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,40,2,128,1,float16,float16,0,0.9692367553710938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,40,8,128,1,float16,float16,0,1.952779197692871
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,40,2,128,1,float16,float16,0,0.509227180480957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,40,40,128,1,float16,float16,0,0.6308911800384521
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,40,4,128,1,float16,float16,0,0.5273663997650146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,40,4,128,1,float16,float16,0,0.9695136070251464
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,40,8,128,1,float16,float16,0,0.5471439838409424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,40,40,128,1,float16,float16,0,0.34834239482879636
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,40,8,128,1,float16,float16,0,0.9962880134582519
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,40,1,128,1,float16,float16,0,0.28441920280456545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,40,2,128,1,float16,float16,0,0.28603999614715575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,40,4,128,1,float16,float16,0,0.30814080238342284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,40,8,128,1,float16,float16,0,0.31130878925323485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,40,40,128,1,float16,float16,0,2.3262847900390624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,40,1,128,1,float16,float16,0,0.4851168155670166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,40,1,128,1,float16,float16,0,4.294660949707032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,40,2,128,1,float16,float16,0,4.255879974365234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,40,40,128,1,float16,float16,0,5.5892894744873045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,40,4,128,1,float16,float16,0,4.211641693115235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,40,1,128,1,float16,float16,0,8.43133316040039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,40,2,128,1,float16,float16,0,8.55100326538086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,40,1,128,1,float16,float16,0,2.086732864379883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,40,4,128,1,float16,float16,0,8.712213134765625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,40,8,128,1,float16,float16,0,8.99404296875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,40,8,128,1,float16,float16,0,4.569414520263672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,40,2,128,1,float16,float16,0,2.093819236755371
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,40,1,128,1,float16,float16,0,1.0789631843566894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,40,2,128,1,float16,float16,0,1.1066831588745116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,40,40,128,1,float16,float16,0,1.414799976348877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,40,4,128,1,float16,float16,0,2.1394655227661135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,40,4,128,1,float16,float16,0,1.1227472305297852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,40,40,128,1,float16,float16,0,0.7442575931549072
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,40,1,128,1,float16,float16,0,0.5715888023376465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,40,8,128,1,float16,float16,0,2.2227296829223633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,40,2,128,1,float16,float16,0,0.5715904235839844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,40,8,128,1,float16,float16,0,1.1445936203002929
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,40,4,128,1,float16,float16,0,0.5844944000244141
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,40,8,128,1,float16,float16,0,0.6136256217956543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,40,40,128,1,float16,float16,0,0.3969759941101074
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,40,2,128,1,float16,float16,0,0.3240256071090698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,40,1,128,1,float16,float16,0,0.31535680294036866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,40,4,128,1,float16,float16,0,0.32204959392547605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,40,1,128,1,float16,float16,0,0.1929919958114624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,40,40,128,1,float16,float16,0,0.23301279544830322
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,40,2,128,1,float16,float16,0,0.17977919578552246
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,40,8,128,1,float16,float16,0,0.3421760082244873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,40,4,128,1,float16,float16,0,0.18944159746170045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,40,40,128,1,float16,float16,0,2.7951871871948244
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,40,8,128,1,float16,float16,0,0.19617600440979005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,40,1,128,1,float16,float16,0,3.8293903350830076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,40,2,128,1,float16,float16,0,4.012430572509766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,40,4,128,1,float16,float16,0,4.07337760925293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,40,40,128,1,float16,float16,0,5.7442176818847654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,40,1,128,1,float16,float16,0,2.019063949584961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,40,40,128,1,float16,float16,0,2.931257629394531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,40,1,128,1,float16,float16,0,8.016286468505859
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,40,4,128,1,float16,float16,0,8.262999725341796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,40,8,128,1,float16,float16,0,4.323915100097656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,40,2,128,1,float16,float16,0,1.9816816329956055
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,40,8,128,1,float16,float16,0,8.716264343261718
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,40,4,128,1,float16,float16,0,2.0715599060058594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,40,8,128,1,float16,float16,0,2.1494768142700194
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,40,40,128,1,float16,float16,0,1.4893376350402832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,40,1,128,1,float16,float16,0,1.0036368370056152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,40,2,128,1,float16,float16,0,1.0578432083129883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,40,40,128,1,float16,float16,0,0.7693935871124268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,40,4,128,1,float16,float16,0,1.0366144180297852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,40,8,128,1,float16,float16,0,1.1058336257934571
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,40,2,128,1,float16,float16,0,0.5296703815460205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,40,40,128,1,float16,float16,0,0.40497279167175293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,40,4,128,1,float16,float16,0,0.5368447780609131
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,40,1,128,1,float16,float16,0,0.2834176063537598
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,40,8,128,1,float16,float16,0,0.5570896148681641
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,40,8,128,1,float16,float16,0,0.306550407409668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,40,1,128,1,float16,float16,0,0.16054400205612182
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,40,2,128,1,float16,float16,0,0.29252800941467283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,40,40,128,1,float16,float16,0,0.22815680503845215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,40,2,128,1,float16,float16,0,0.16795680522918702
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,40,8,128,1,float16,float16,0,0.1780608057975769
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,40,40,128,1,float16,float16,0,0.1405168056488037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,40,1,128,1,float16,float16,0,0.1021232008934021
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,40,2,128,1,float16,float16,0,8.268540954589843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,40,2,128,1,float16,float16,0,0.10264960527420045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,40,4,128,1,float16,float16,0,0.10924320220947266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,40,8,128,1,float16,float16,0,0.10777759552001953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,40,1,128,1,float16,float16,0,0.5535264015197754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,40,4,128,1,float16,float16,0,0.29250240325927734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,40,4,128,1,float16,float16,0,0.16928800344467163
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,40,1,128,1,float16,float16,0,4.761505508422852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,40,2,128,1,float16,float16,0,4.926820755004883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,40,1,128,1,float16,float16,0,2.2496400833129884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,40,2,128,1,float16,float16,0,2.3536848068237304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,40,4,128,1,float16,float16,0,2.451639938354492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,40,8,128,1,float16,float16,0,2.715764808654785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,40,4,128,1,float16,float16,0,4.974065780639648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,40,40,128,1,float16,float16,0,4.181726455688477
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,40,8,128,1,float16,float16,0,5.411531066894531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,40,1,128,1,float16,float16,0,0.6170207977294921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,40,2,128,1,float16,float16,0,0.5971759796142578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,40,2,128,1,float16,float16,0,1.176806354522705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,40,1,128,1,float16,float16,0,1.152500820159912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,40,4,128,1,float16,float16,0,1.2317312240600586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,40,40,128,1,float16,float16,0,1.0839695930480957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,40,8,128,1,float16,float16,0,1.3570096015930175
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,40,1,128,1,float16,float16,0,0.3165616035461426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,40,4,128,1,float16,float16,0,0.6331632137298584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,40,2,128,1,float16,float16,0,0.32412641048431395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,40,40,128,1,float16,float16,0,0.5584559917449952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,40,4,128,1,float16,float16,0,0.32960000038146975
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,40,8,128,1,float16,float16,0,0.6679999828338623
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,40,8,128,1,float16,float16,0,0.36636159420013426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,40,40,128,1,float16,float16,0,2.086163139343262
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,40,1,128,1,float16,float16,0,0.17381919622421266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,40,40,128,1,float16,float16,0,0.2971487998962402
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,40,2,128,1,float16,float16,0,0.17742880582809448
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,40,40,128,1,float16,float16,0,0.16834720373153686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,40,4,128,1,float16,float16,0,0.18488800525665283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,40,1,128,1,float16,float16,0,0.10585440397262573
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,40,40,128,1,float16,float16,0,0.09737439751625061
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,40,8,128,1,float16,float16,0,0.11384960412979125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,40,1,128,1,float16,float16,0,0.06552479863166809
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,40,2,128,1,float16,float16,0,0.10736639499664306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,40,2,128,1,float16,float16,0,0.062249600887298584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,40,40,128,1,float16,float16,0,0.06115040183067322
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,40,4,128,1,float16,float16,0,0.06706079840660095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,40,8,128,1,float16,float16,0,0.07456960082054138
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,40,8,128,1,float16,float16,0,0.04365279972553253
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,40,4,128,1,float16,float16,0,0.044121599197387694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,40,2,128,1,float16,float16,0,1.5837087631225586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,40,1,128,1,float16,float16,0,1.5304207801818848
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,40,4,128,1,float16,float16,0,1.6524063110351563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,40,8,128,1,float16,float16,0,1.8881343841552733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,40,1,128,1,float16,float16,0,0.8031727790832519
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,40,8,128,1,float16,float16,0,0.19477440118789674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,40,4,128,1,float16,float16,0,0.1074336051940918
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,40,2,128,1,float16,float16,0,0.8281536102294922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,40,1,128,1,float16,float16,0,0.039585599303245546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,40,4,128,1,float16,float16,0,0.8644960403442383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,40,2,128,1,float16,float16,0,0.04508799910545349
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,40,1,128,1,float16,float16,0,0.40007519721984863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,40,40,128,1,float16,float16,0,1.8123952865600585
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,40,2,128,1,float16,float16,0,0.4207312107086182
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,40,8,128,1,float16,float16,0,0.9629887580871582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,40,4,128,1,float16,float16,0,0.4298111915588379
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,40,1,128,1,float16,float16,0,0.21030240058898925
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,40,4,128,1,float16,float16,0,0.231060791015625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,40,40,128,1,float16,float16,0,0.4710512161254883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,40,1,128,1,float16,float16,0,0.11666239500045776
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,40,8,128,1,float16,float16,0,0.2598560094833374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,40,2,128,1,float16,float16,0,0.12183359861373902
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,40,8,128,1,float16,float16,0,0.13912800550460816
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,40,1,128,1,float16,float16,0,0.0711072027683258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,40,2,128,1,float16,float16,0,0.07195519804954528
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,40,4,128,1,float16,float16,0,0.076009601354599
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,40,8,128,1,float16,float16,0,0.08127840161323548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,40,40,128,1,float16,float16,0,0.07461119890213012
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,40,1,128,1,float16,float16,0,0.045003199577331544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,40,2,128,1,float16,float16,0,0.04494400024414062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,40,4,128,1,float16,float16,0,0.04574880003929138
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,40,8,128,1,float16,float16,0,0.051868802309036253
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,40,40,128,1,float16,float16,0,0.0442656010389328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,40,1,128,1,float16,float16,0,0.030131199955940248
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,40,2,128,1,float16,float16,0,0.028886398673057555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,40,4,128,1,float16,float16,0,0.030248001217842102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,40,8,128,1,float16,float16,0,0.03215999901294708
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,40,40,128,1,float16,float16,0,0.027219200134277345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,40,1,128,1,float16,float16,0,0.022908799350261688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,40,2,128,1,float16,float16,0,0.023343999683856965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,40,4,128,1,float16,float16,0,0.023233599960803986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,40,8,128,1,float16,float16,0,0.024382400512695312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,40,1,128,1,float16,float16,0,0.5921232223510742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,40,2,128,1,float16,float16,0,0.6231391906738282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,40,40,128,1,float16,float16,0,0.9109760284423828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,40,8,128,1,float16,float16,0,0.4934351921081543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,40,2,128,1,float16,float16,0,0.22328319549560546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,40,4,128,1,float16,float16,0,0.6730175971984863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,40,8,128,1,float16,float16,0,0.7923984050750732
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,40,1,128,1,float16,float16,0,0.3082976102828979
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,40,4,128,1,float16,float16,0,0.12600159645080566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,40,2,128,1,float16,float16,0,0.3241791963577271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,40,4,128,1,float16,float16,0,0.3520672082901001
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,40,40,128,1,float16,float16,0,0.8823200225830078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,40,1,128,1,float16,float16,0,0.16529279947280884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,40,8,128,1,float16,float16,0,0.4086944103240967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,40,40,128,1,float16,float16,0,0.4548448085784912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,40,4,128,1,float16,float16,0,0.18805439472198487
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,40,8,128,1,float16,float16,0,0.22177441120147706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,40,40,128,1,float16,float16,0,0.24224319458007812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,40,2,128,1,float16,float16,0,0.0959984004497528
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,40,4,128,1,float16,float16,0,0.1040768027305603
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,40,40,128,1,float16,float16,0,0.13004640340805054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,40,8,128,1,float16,float16,0,0.12215839624404908
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,40,1,128,1,float16,float16,0,0.056883198022842404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,40,2,128,1,float16,float16,0,0.05781440138816833
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,40,4,128,1,float16,float16,0,0.060063999891281125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,40,8,128,1,float16,float16,0,0.06795039772987366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,40,40,128,1,float16,float16,0,0.07117279767990112
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,40,1,128,1,float16,float16,0,0.03334240019321442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,40,2,128,1,float16,float16,0,0.03407039940357208
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,40,4,128,1,float16,float16,0,0.034923198819160464
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,40,8,128,1,float16,float16,0,0.04125120043754578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,40,40,128,1,float16,float16,0,0.03640320003032684
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,40,1,128,1,float16,float16,0,0.02295999974012375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,40,4,128,1,float16,float16,0,0.02319519966840744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,40,8,128,1,float16,float16,0,0.024040000140666963
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,40,40,128,1,float16,float16,0,0.021587200462818146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,40,1,128,1,float16,float16,0,0.01809599995613098
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,40,2,128,1,float16,float16,0,0.018167999386787415
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,40,4,128,1,float16,float16,0,0.018404799699783325
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,40,8,128,1,float16,float16,0,0.01910399943590164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,40,40,128,1,float16,float16,0,0.020212799310684204
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,40,1,128,1,float16,float16,0,0.017836800217628478
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,40,2,128,1,float16,float16,0,0.017772799730300902
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,40,4,128,1,float16,float16,0,0.017878399789333345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,40,8,128,1,float16,float16,0,0.018433600664138794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,40,40,128,1,float16,float16,0,0.25220959186553954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,40,2,128,1,float16,float16,0,0.3118144035339355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,40,40,128,1,float16,float16,0,0.14470399618148805
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,40,4,128,1,float16,float16,0,0.35028159618377686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,40,8,128,1,float16,float16,0,0.42005600929260256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,40,2,128,1,float16,float16,0,0.17161279916763306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,40,1,128,1,float16,float16,0,0.1659216046333313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,40,40,128,1,float16,float16,0,0.4535183906555176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,40,2,128,1,float16,float16,0,0.17017600536346436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,40,4,128,1,float16,float16,0,0.1865231990814209
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,40,1,128,1,float16,float16,0,0.09435679912567138
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,40,1,128,1,float16,float16,0,0.09023360013961793
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,40,8,128,1,float16,float16,0,0.21848320960998535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,40,4,128,1,float16,float16,0,0.10315840244293213
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,40,2,128,1,float16,float16,0,0.09477760195732117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,40,40,128,1,float16,float16,0,0.2383280038833618
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,40,1,128,1,float16,float16,0,0.05308160185813904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,40,8,128,1,float16,float16,0,0.11940480470657348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,40,2,128,1,float16,float16,0,0.05384320020675659
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,40,40,128,1,float16,float16,0,0.13028160333633423
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,40,4,128,1,float16,float16,0,0.05813760161399841
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,40,8,128,1,float16,float16,0,0.06800159811973572
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,40,1,128,1,float16,float16,0,0.02945440113544464
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,40,40,128,1,float16,float16,0,0.07004799842834472
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,40,2,128,1,float16,float16,0,0.02948000133037567
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,40,4,128,1,float16,float16,0,0.030225598812103273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,40,8,128,1,float16,float16,0,0.035979199409484866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,40,40,128,1,float16,float16,0,0.03775199949741363
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,40,2,128,1,float16,float16,0,0.022864000499248506
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,40,1,128,1,float16,float16,0,0.018780800700187682
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,40,40,128,1,float16,float16,0,0.02027679979801178
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,40,2,128,1,float16,float16,0,0.01911199986934662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,40,4,128,1,float16,float16,0,0.019206400215625762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,40,8,128,1,float16,float16,0,0.02033119946718216
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,40,4,128,1,float16,float16,0,0.01605760008096695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,40,40,128,1,float16,float16,0,0.017000000178813934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,40,8,128,1,float16,float16,0,0.01709119975566864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,40,2,128,1,float16,float16,0,0.015171200037002563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,40,4,128,1,float16,float16,0,0.01523360013961792
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,40,8,128,1,float16,float16,0,0.01584160029888153
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,40,40,128,1,float16,float16,0,0.015881599485874177
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,40,2,128,1,float16,float16,0,0.0152879998087883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,40,4,128,1,float16,float16,0,0.015143999457359314
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,40,8,128,1,float16,float16,0,0.01547040045261383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,40,2,128,1,float16,float16,0,0.16204639673233032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,40,1,128,1,float16,float16,0,0.15554879903793334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,40,4,128,1,float16,float16,0,0.1887168049812317
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,40,8,128,1,float16,float16,0,0.2384495973587036
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,40,1,128,1,float16,float16,0,0.09005280137062073
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,40,40,128,1,float16,float16,0,0.30475358963012694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,40,2,128,1,float16,float16,0,0.09155200123786926
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,40,8,128,1,float16,float16,0,0.13013440370559692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,40,4,128,1,float16,float16,0,0.10493279695510864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,40,1,128,1,float16,float16,0,0.05339199900627136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,40,40,128,1,float16,float16,0,0.159062397480011
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,40,4,128,1,float16,float16,0,0.0620464026927948
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,40,2,128,1,float16,float16,0,0.05570399761199951
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,40,8,128,1,float16,float16,0,0.0748304009437561
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,40,40,128,1,float16,float16,0,0.08454239964485169
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,40,2,128,1,float16,float16,0,0.02934719920158386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,40,1,128,1,float16,float16,0,0.029844799637794496
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,40,4,128,1,float16,float16,0,0.030950400233268737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,40,8,128,1,float16,float16,0,0.04243040084838867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,40,1,128,1,float16,float16,0,0.018651199340820313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,40,1,128,1,float16,float16,0,0.01598079949617386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,40,2,128,1,float16,float16,0,0.018555200099945067
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,40,40,128,1,float16,float16,0,0.04579679965972901
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,40,4,128,1,float16,float16,0,0.019843199849128725
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,40,8,128,1,float16,float16,0,0.025135999917984007
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,40,40,128,1,float16,float16,0,0.027187201380729675
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,40,1,128,1,float16,float16,0,0.015961599349975587
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,40,2,128,1,float16,float16,0,0.0158160001039505
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,40,1,128,1,float16,float16,0,0.015328000485897064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,40,4,128,1,float16,float16,0,0.016459199786186218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,40,8,128,1,float16,float16,0,0.018731200695037843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,40,40,128,1,float16,float16,0,0.018731200695037843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,40,1,128,1,float16,float16,0,0.015222400426864624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,40,2,128,1,float16,float16,0,0.015067200362682342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,40,4,128,1,float16,float16,0,0.015396800637245179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,40,8,128,1,float16,float16,0,0.015647999942302704
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,40,1,128,1,float16,float16,0,0.29792799949646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,40,40,128,1,float16,float16,0,0.01615999937057495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,40,1,128,1,float16,float16,0,0.014987200498580933
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,40,2,128,1,float16,float16,0,0.015167999267578124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,40,4,128,1,float16,float16,0,0.015192000567913056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,40,40,128,1,float16,float16,0,0.015758399665355683
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,40,1,128,1,float16,float16,0,0.014819200336933135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,40,2,128,1,float16,float16,0,0.014715200662612915
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,40,4,128,1,float16,float16,0,0.015056000649929046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,40,8,128,1,float16,float16,0,0.015145599842071533
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,40,8,128,1,float16,float16,0,0.015060800313949584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,40,1,128,1,float16,float16,0,0.08985919952392578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,40,2,128,1,float16,float16,0,0.0966368019580841
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,40,1,128,1,float16,float16,0,0.054388797283172606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,40,8,128,1,float16,float16,0,0.128603196144104
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,40,2,128,1,float16,float16,0,0.05939520001411438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,40,40,128,1,float16,float16,0,0.24644479751586915
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,40,4,128,1,float16,float16,0,0.06942239999771119
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,40,8,128,1,float16,float16,0,0.07382879853248596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,40,1,128,1,float16,float16,0,0.02983199954032898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,40,40,128,1,float16,float16,0,0.12899839878082275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,40,4,128,1,float16,float16,0,0.03835040032863617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,40,8,128,1,float16,float16,0,0.041766399145126344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,40,40,128,1,float16,float16,0,0.0674239993095398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,40,2,128,1,float16,float16,0,0.019696000218391418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,40,4,128,1,float16,float16,0,0.025547200441360475
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,40,8,128,1,float16,float16,0,0.025672000646591187
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,40,40,128,1,float16,float16,0,0.03537440001964569
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,40,1,128,1,float16,float16,0,0.01581919938325882
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,40,2,128,1,float16,float16,0,0.016067199409008026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,40,4,128,1,float16,float16,0,0.018060800433158875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,40,8,128,1,float16,float16,0,0.018035200238227845
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,40,40,128,1,float16,float16,0,0.025123199820518492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,40,1,128,1,float16,float16,0,0.015222400426864624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,40,2,128,1,float16,float16,0,0.014990399777889251
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,40,4,128,1,float16,float16,0,0.01541599929332733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,40,8,128,1,float16,float16,0,0.015249599516391755
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,40,40,128,1,float16,float16,0,0.017977599799633027
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,40,1,128,1,float16,float16,0,0.014920000731945039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,40,2,128,1,float16,float16,0,0.015116800367832185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,40,4,128,1,float16,float16,0,0.014892800152301789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,40,8,128,1,float16,float16,0,0.01496960073709488
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,40,2,128,1,float16,float16,0,0.015851199626922607
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,40,40,128,1,float16,float16,0,0.015718400478363037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,40,1,128,1,float16,float16,0,0.014900800585746766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,40,4,128,1,float16,float16,0,0.014868800342082978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,40,2,128,1,float16,float16,0,0.014638400077819825
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,40,8,128,1,float16,float16,0,0.014696000516414643
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,40,40,128,1,float16,float16,0,0.015611200034618378
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,40,2,128,1,float16,float16,0,0.014748799800872802
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,40,4,128,1,float16,float16,0,0.014820800721645355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,40,8,128,1,float16,float16,0,0.014574399590492249
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,40,1,128,1,float16,float16,0,0.02680320143699646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,40,2,128,1,float16,float16,0,0.03433600068092346
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,40,1,128,1,float16,float16,0,0.01507679969072342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,40,4,128,1,float16,float16,0,0.05186399817466736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,40,8,128,1,float16,float16,0,0.0892687976360321
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,40,1,128,1,float16,float16,0,0.018880000710487364
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,40,2,128,1,float16,float16,0,0.024695999920368195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,40,40,128,1,float16,float16,0,0.1899616003036499
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,40,4,128,1,float16,float16,0,0.03255999982357025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,40,8,128,1,float16,float16,0,0.050147199630737306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,40,1,128,1,float16,float16,0,0.015880000591278077
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,40,4,128,1,float16,float16,0,0.11906239986419678
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,40,40,128,1,float16,float16,0,0.1008463978767395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,40,4,128,1,float16,float16,0,0.023563200235366823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,40,2,128,1,float16,float16,0,0.017689600586891174
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,40,40,128,1,float16,float16,0,0.05579360127449036
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,40,8,128,1,float16,float16,0,0.031411200761795044
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,40,2,128,1,float16,float16,0,0.015004800260066986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,40,4,128,1,float16,float16,0,0.01690240055322647
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,40,8,128,1,float16,float16,0,0.023049600422382355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,40,40,128,1,float16,float16,0,0.03295199871063233
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,40,1,128,1,float16,float16,0,0.01414719969034195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,40,2,128,1,float16,float16,0,0.01454080045223236
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,40,2,128,1,float16,float16,0,0.030715200304985046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,40,4,128,1,float16,float16,0,0.014587199687957764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,40,8,128,1,float16,float16,0,0.01623679995536804
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,40,1,128,1,float16,float16,0,0.018915200233459474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,40,40,128,1,float16,float16,0,0.0234592005610466
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,40,2,128,1,float16,float16,0,0.014371199905872345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,40,8,128,1,float16,float16,0,0.014372800290584565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,40,40,128,1,float16,float16,0,0.01727679967880249
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,40,1,128,1,float16,float16,0,0.014295999705791474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,40,2,128,1,float16,float16,0,0.013923199474811554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,40,4,128,1,float16,float16,0,0.014387199282646179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,40,8,128,1,float16,float16,0,0.014374400675296783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,40,40,128,1,float16,float16,0,0.015440000593662262
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,40,1,128,1,float16,float16,0,0.014099200069904328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,40,2,128,1,float16,float16,0,0.014302399754524232
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,40,4,128,1,float16,float16,0,0.014099200069904328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,40,8,128,1,float16,float16,0,0.014321599900722504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,40,40,128,1,float16,float16,0,0.015115199983119965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,40,1,128,1,float16,float16,0,0.014168000221252442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,40,2,128,1,float16,float16,0,0.013840000331401824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,40,4,128,1,float16,float16,0,0.014008000493049622
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,40,8,128,1,float16,float16,0,0.01385599970817566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,40,1,128,1,float16,float16,0,0.014683200418949128
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,40,1,128,1,float16,float16,0,0.014881600439548493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,40,4,128,1,float16,float16,0,0.014177599549293518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,40,1,128,1,float16,float16,0,0.01404000073671341
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,32,1,128,1,float16,float16,0,42.34560546875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,32,4,128,1,float16,float16,0,41.876129150390625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,32,2,128,1,float16,float16,0,43.019012451171875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,32,1,128,1,float16,float16,0,21.57495880126953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,32,1,128,1,float16,float16,0,84.422216796875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,32,2,128,1,float16,float16,0,85.2396484375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,32,4,128,1,float16,float16,0,85.22312622070312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,32,8,128,1,float16,float16,0,85.30680541992187
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,32,8,128,1,float16,float16,0,42.76105651855469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,32,2,128,1,float16,float16,0,21.67595672607422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,32,1,128,1,float16,float16,0,10.73605728149414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,32,2,128,1,float16,float16,0,10.714959716796875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,32,4,128,1,float16,float16,0,10.973104095458984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,32,4,128,1,float16,float16,0,21.325296020507814
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,32,8,128,1,float16,float16,0,21.646871948242186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,32,8,128,1,float16,float16,0,10.923117065429688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,32,1,128,1,float16,float16,0,47.69443969726562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,32,2,128,1,float16,float16,0,48.554629516601565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,32,4,128,1,float16,float16,0,48.089413452148435
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,32,8,128,1,float16,float16,0,48.763580322265625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,32,1,128,1,float16,float16,0,23.97693786621094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,32,2,128,1,float16,float16,0,24.407762145996095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,32,4,128,1,float16,float16,0,24.17243194580078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,32,8,128,1,float16,float16,0,24.417446899414063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,32,1,128,1,float16,float16,0,12.29598388671875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,32,2,128,1,float16,float16,0,12.076953887939453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,32,4,128,1,float16,float16,0,12.412260437011719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,32,1,128,1,float16,float16,0,6.106371307373047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,32,8,128,1,float16,float16,0,12.477667236328125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,32,2,128,1,float16,float16,0,6.141142272949219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,32,4,128,1,float16,float16,0,6.2604927062988285
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,32,8,128,1,float16,float16,0,6.314932632446289
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,32,1,128,1,float16,float16,0,33.861138916015626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,32,2,128,1,float16,float16,0,33.743649291992185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,32,32,128,1,float16,float16,0,6.492976379394531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,32,32,128,1,float16,float16,0,11.485858917236328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,32,32,128,1,float16,float16,0,12.762617492675782
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,32,32,128,1,float16,float16,0,22.018243408203126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,32,32,128,1,float16,float16,0,17.775494384765626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,32,1,128,1,float16,float16,0,16.710292053222656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,32,2,128,1,float16,float16,0,16.650698852539062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,32,32,128,1,float16,float16,0,25.360304260253905
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,32,4,128,1,float16,float16,0,34.11680603027344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,32,8,128,1,float16,float16,0,33.8828857421875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,32,32,128,1,float16,float16,0,9.157615661621094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,32,1,128,1,float16,float16,0,8.47974853515625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,32,2,128,1,float16,float16,0,8.423375701904297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,32,4,128,1,float16,float16,0,8.623654174804688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,32,4,128,1,float16,float16,0,16.595870971679688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,32,32,128,1,float16,float16,0,44.0778564453125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,32,8,128,1,float16,float16,0,17.192498779296876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,32,32,128,1,float16,float16,0,4.607817459106445
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,32,1,128,1,float16,float16,0,4.265174484252929
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,32,2,128,1,float16,float16,0,4.259379196166992
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,32,4,128,1,float16,float16,0,4.391526412963867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,32,8,128,1,float16,float16,0,8.920540618896485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,32,8,128,1,float16,float16,0,4.33015022277832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,32,2,128,1,float16,float16,0,21.596206665039062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,32,1,128,1,float16,float16,0,21.775900268554686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,32,4,128,1,float16,float16,0,21.274708557128907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,32,32,128,1,float16,float16,0,23.10259094238281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,32,1,128,1,float16,float16,0,10.598753356933594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,32,32,128,1,float16,float16,0,11.854825592041015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,32,2,128,1,float16,float16,0,10.841505432128907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,32,1,128,1,float16,float16,0,42.492208862304686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,32,32,128,1,float16,float16,0,5.932673645019531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,32,2,128,1,float16,float16,0,42.9076904296875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,32,4,128,1,float16,float16,0,43.469180297851565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,32,8,128,1,float16,float16,0,43.404296875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,32,8,128,1,float16,float16,0,22.199496459960937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,32,4,128,1,float16,float16,0,10.686115264892578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,32,1,128,1,float16,float16,0,5.376808166503906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,32,8,128,1,float16,float16,0,11.161804962158204
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,32,2,128,1,float16,float16,0,5.459179306030274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,32,32,128,1,float16,float16,0,3.0615615844726562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,32,4,128,1,float16,float16,0,5.503615951538086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,32,1,128,1,float16,float16,0,2.792363166809082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,32,2,128,1,float16,float16,0,2.768627166748047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,32,8,128,1,float16,float16,0,5.615528106689453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,32,4,128,1,float16,float16,0,2.8050512313842773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,32,8,128,1,float16,float16,0,2.9433727264404297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,32,1,128,1,float16,float16,0,12.28804931640625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,32,2,128,1,float16,float16,0,12.150198364257813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,32,32,128,1,float16,float16,0,13.205992126464844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,32,4,128,1,float16,float16,0,12.568694305419921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,32,1,128,1,float16,float16,0,6.045732879638672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,32,32,128,1,float16,float16,0,6.705894470214844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,32,2,128,1,float16,float16,0,6.243339157104492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,32,1,128,1,float16,float16,0,24.25872802734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,32,2,128,1,float16,float16,0,24.269427490234374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,32,4,128,1,float16,float16,0,24.633882141113283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,32,8,128,1,float16,float16,0,25.071961975097658
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,32,8,128,1,float16,float16,0,12.631400299072265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,32,4,128,1,float16,float16,0,6.143999862670898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,32,8,128,1,float16,float16,0,6.219929504394531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,32,32,128,1,float16,float16,0,3.461172866821289
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,32,1,128,1,float16,float16,0,3.104335975646973
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,32,2,128,1,float16,float16,0,3.1230352401733397
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,32,4,128,1,float16,float16,0,3.1799264907836915
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,32,32,128,1,float16,float16,0,1.777364730834961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,32,1,128,1,float16,float16,0,1.6014768600463867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,32,8,128,1,float16,float16,0,3.2250431060791014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,32,2,128,1,float16,float16,0,1.6409168243408203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,32,4,128,1,float16,float16,0,1.6816543579101562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,32,8,128,1,float16,float16,0,1.7208080291748047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,32,1,128,1,float16,float16,0,11.132377624511719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,32,2,128,1,float16,float16,0,11.097958374023438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,32,4,128,1,float16,float16,0,11.356102752685548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,32,32,128,1,float16,float16,0,12.59574203491211
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,32,1,128,1,float16,float16,0,5.606208038330078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,32,32,128,1,float16,float16,0,6.3339599609375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,32,2,128,1,float16,float16,0,5.554110336303711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,32,2,128,1,float16,float16,0,21.852085876464844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,32,1,128,1,float16,float16,0,22.219261169433594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,32,4,128,1,float16,float16,0,22.190516662597656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,32,32,128,1,float16,float16,0,3.1820688247680664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,32,8,128,1,float16,float16,0,11.295750427246094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,32,8,128,1,float16,float16,0,22.79381103515625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,32,1,128,1,float16,float16,0,2.7335039138793946
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,32,4,128,1,float16,float16,0,5.696121597290039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,32,2,128,1,float16,float16,0,2.8288368225097655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,32,1,128,1,float16,float16,0,1.4103712081909179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,32,32,128,1,float16,float16,0,1.6387823104858399
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,32,4,128,1,float16,float16,0,2.8096975326538085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,32,8,128,1,float16,float16,0,2.8874048233032226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,32,2,128,1,float16,float16,0,1.4553664207458497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,32,32,128,1,float16,float16,0,0.9077887535095215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,32,1,128,1,float16,float16,0,0.7591455936431885
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,32,4,128,1,float16,float16,0,1.5135760307312012
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,32,2,128,1,float16,float16,0,0.800160026550293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,32,8,128,1,float16,float16,0,1.5250144004821777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,32,4,128,1,float16,float16,0,0.8262751579284668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,32,8,128,1,float16,float16,0,0.8141183853149414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,32,8,128,1,float16,float16,0,5.679916763305664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,32,1,128,1,float16,float16,0,6.3152320861816404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,32,2,128,1,float16,float16,0,6.263281631469726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,32,32,128,1,float16,float16,0,7.511217498779297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,32,4,128,1,float16,float16,0,6.4353889465332035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,32,32,128,1,float16,float16,0,3.7111377716064453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,32,1,128,1,float16,float16,0,3.1825008392333984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,32,2,128,1,float16,float16,0,3.18939208984375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,32,2,128,1,float16,float16,0,12.651350402832032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,32,1,128,1,float16,float16,0,12.734356689453126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,32,4,128,1,float16,float16,0,13.062875366210937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,32,8,128,1,float16,float16,0,12.964894104003907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,32,8,128,1,float16,float16,0,6.570796966552734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,32,32,128,1,float16,float16,0,1.9209856033325194
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,32,1,128,1,float16,float16,0,1.600289535522461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,32,2,128,1,float16,float16,0,1.656991958618164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,32,4,128,1,float16,float16,0,3.2043888092041017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,32,32,128,1,float16,float16,0,1.0163359642028809
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,32,1,128,1,float16,float16,0,0.8458975791931153
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,32,8,128,1,float16,float16,0,3.31541748046875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,32,4,128,1,float16,float16,0,1.6795520782470703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,32,2,128,1,float16,float16,0,0.8589936256408691
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,32,32,128,1,float16,float16,0,0.5567952156066894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,32,4,128,1,float16,float16,0,0.9039567947387696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,32,1,128,1,float16,float16,0,0.4646399974822998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,32,2,128,1,float16,float16,0,0.4908720016479492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,32,8,128,1,float16,float16,0,0.906833553314209
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,32,4,128,1,float16,float16,0,0.5041696071624756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,32,8,128,1,float16,float16,0,0.49907522201538085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,32,8,128,1,float16,float16,0,1.6741903305053711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,32,1,128,1,float16,float16,0,5.90523681640625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,32,2,128,1,float16,float16,0,5.8757072448730465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,32,32,128,1,float16,float16,0,7.189950561523437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,32,4,128,1,float16,float16,0,5.866702270507813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,32,32,128,1,float16,float16,0,3.6266223907470705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,32,1,128,1,float16,float16,0,2.880625534057617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,32,1,128,1,float16,float16,0,11.66812973022461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,32,2,128,1,float16,float16,0,11.792972564697266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,32,4,128,1,float16,float16,0,11.916252899169923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,32,8,128,1,float16,float16,0,12.307592010498047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,32,8,128,1,float16,float16,0,6.200563049316406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,32,4,128,1,float16,float16,0,2.9679216384887694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,32,1,128,1,float16,float16,0,1.4651200294494628
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,32,32,128,1,float16,float16,0,1.8502288818359376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,32,8,128,1,float16,float16,0,3.02685432434082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,32,2,128,1,float16,float16,0,1.485979175567627
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,32,4,128,1,float16,float16,0,1.5470687866210937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,32,1,128,1,float16,float16,0,0.7536655902862549
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,32,8,128,1,float16,float16,0,1.5707311630249023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,32,32,128,1,float16,float16,0,0.975438404083252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,32,2,128,1,float16,float16,0,0.7879151821136474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,32,4,128,1,float16,float16,0,0.811184024810791
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,32,8,128,1,float16,float16,0,0.805083179473877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,32,32,128,1,float16,float16,0,0.5249055862426758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,32,1,128,1,float16,float16,0,0.4100287914276123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,32,4,128,1,float16,float16,0,0.4382688045501709
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,32,2,128,1,float16,float16,0,0.43627519607543946
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,32,32,128,1,float16,float16,0,0.299068808555603
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,32,1,128,1,float16,float16,0,0.24721119403839112
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,32,2,128,1,float16,float16,0,0.2483680009841919
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,32,4,128,1,float16,float16,0,0.2531136035919189
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,32,8,128,1,float16,float16,0,0.2657360076904297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,32,2,128,1,float16,float16,0,2.8982336044311525
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,32,8,128,1,float16,float16,0,0.441315221786499
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,32,1,128,1,float16,float16,0,3.295671844482422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,32,32,128,1,float16,float16,0,4.523484802246093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,32,1,128,1,float16,float16,0,6.697090911865234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,32,2,128,1,float16,float16,0,6.897483062744141
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,32,2,128,1,float16,float16,0,3.339913558959961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,32,4,128,1,float16,float16,0,7.004027557373047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,32,8,128,1,float16,float16,0,7.185062408447266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,32,4,128,1,float16,float16,0,3.444308853149414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,32,8,128,1,float16,float16,0,3.626731109619141
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,32,1,128,1,float16,float16,0,1.7179855346679687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,32,2,128,1,float16,float16,0,1.7292976379394531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,32,32,128,1,float16,float16,0,2.233527946472168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,32,4,128,1,float16,float16,0,1.758812713623047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,32,32,128,1,float16,float16,0,1.1415712356567382
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,32,8,128,1,float16,float16,0,1.8029584884643555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,32,2,128,1,float16,float16,0,0.8875663757324219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,32,1,128,1,float16,float16,0,0.9116944313049317
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,32,4,128,1,float16,float16,0,0.9017663955688476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,32,1,128,1,float16,float16,0,0.4624591827392578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,32,32,128,1,float16,float16,0,0.6105823993682862
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,32,8,128,1,float16,float16,0,0.9263039588928222
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,32,2,128,1,float16,float16,0,0.4842112064361572
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,32,1,128,1,float16,float16,0,0.24284639358520507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,32,32,128,1,float16,float16,0,0.3367167949676514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,32,2,128,1,float16,float16,0,0.27211039066314696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,32,8,128,1,float16,float16,0,0.2809423923492432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,32,4,128,1,float16,float16,0,0.26960000991821287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,32,32,128,1,float16,float16,0,0.19720640182495117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,32,1,128,1,float16,float16,0,0.15628160238265992
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,32,2,128,1,float16,float16,0,0.1648975968360901
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,32,4,128,1,float16,float16,0,0.16191200017929078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,32,8,128,1,float16,float16,0,0.17575839757919312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,32,4,128,1,float16,float16,0,0.4808527946472168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,32,8,128,1,float16,float16,0,0.5034351825714112
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,32,1,128,1,float16,float16,0,3.150823974609375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,32,32,128,1,float16,float16,0,4.571539306640625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,32,2,128,1,float16,float16,0,3.1836896896362306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,32,1,128,1,float16,float16,0,6.366161727905274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,32,2,128,1,float16,float16,0,6.458927917480469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,32,4,128,1,float16,float16,0,6.690968322753906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,32,1,128,1,float16,float16,0,1.6180416107177735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,32,8,128,1,float16,float16,0,6.886094665527343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,32,32,128,1,float16,float16,0,2.3182687759399414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,32,8,128,1,float16,float16,0,3.413840103149414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,32,4,128,1,float16,float16,0,0.8480192184448242
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,32,32,128,1,float16,float16,0,1.1625935554504394
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,32,2,128,1,float16,float16,0,0.8187151908874511
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,32,2,128,1,float16,float16,0,1.5684320449829101
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,32,4,128,1,float16,float16,0,1.6042303085327148
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,32,8,128,1,float16,float16,0,1.6862127304077148
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,32,1,128,1,float16,float16,0,0.42569122314453123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,32,32,128,1,float16,float16,0,0.6018064022064209
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,32,2,128,1,float16,float16,0,0.4406112194061279
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,32,8,128,1,float16,float16,0,0.8723872184753418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,32,4,128,1,float16,float16,0,0.4506976127624512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,32,4,128,1,float16,float16,0,3.2963790893554688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,32,8,128,1,float16,float16,0,0.467193603515625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,32,32,128,1,float16,float16,0,0.3289727926254272
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,32,1,128,1,float16,float16,0,0.23483679294586182
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,32,2,128,1,float16,float16,0,0.24508960247039796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,32,4,128,1,float16,float16,0,0.2457200050354004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,32,1,128,1,float16,float16,0,0.1388479948043823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,32,2,128,1,float16,float16,0,0.13798079490661622
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,32,4,128,1,float16,float16,0,0.14691200256347656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,32,32,128,1,float16,float16,0,0.19385279417037965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,32,8,128,1,float16,float16,0,0.15116000175476074
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,32,32,128,1,float16,float16,0,0.11530400514602661
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,32,2,128,1,float16,float16,0,0.09898239970207215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,32,4,128,1,float16,float16,0,0.09917280077934265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,32,1,128,1,float16,float16,0,0.09352160096168519
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,32,8,128,1,float16,float16,0,0.09635840058326721
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,32,1,128,1,float16,float16,0,0.8026479721069336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,32,1,128,1,float16,float16,0,1.8552879333496093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,32,8,128,1,float16,float16,0,0.2566224098205566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,32,1,128,1,float16,float16,0,3.781372833251953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,32,2,128,1,float16,float16,0,1.872960090637207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,32,32,128,1,float16,float16,0,3.371267318725586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,32,4,128,1,float16,float16,0,3.949630355834961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,32,8,128,1,float16,float16,0,4.347732925415039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,32,4,128,1,float16,float16,0,2.0386688232421877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,32,1,128,1,float16,float16,0,0.9597392082214355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,32,2,128,1,float16,float16,0,0.9740719795227051
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,32,4,128,1,float16,float16,0,1.0108752250671387
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,32,8,128,1,float16,float16,0,2.108673667907715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,32,32,128,1,float16,float16,0,0.8857999801635742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,32,32,128,1,float16,float16,0,1.6997440338134766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,32,1,128,1,float16,float16,0,0.49891037940979005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,32,2,128,1,float16,float16,0,0.5054768085479736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,32,4,128,1,float16,float16,0,0.5216512203216552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,32,8,128,1,float16,float16,0,0.5695119857788086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,32,1,128,1,float16,float16,0,0.26170079708099364
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,32,2,128,1,float16,float16,0,0.2693759918212891
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,32,32,128,1,float16,float16,0,0.4593935966491699
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,32,4,128,1,float16,float16,0,0.27820160388946535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,32,8,128,1,float16,float16,0,0.2970031976699829
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,32,1,128,1,float16,float16,0,0.1498128056526184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,32,32,128,1,float16,float16,0,0.24874560832977294
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,32,2,128,1,float16,float16,0,0.15066080093383788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,32,4,128,1,float16,float16,0,0.15523040294647217
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,32,32,128,1,float16,float16,0,0.13849279880523682
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,32,2,128,1,float16,float16,0,0.09292960166931152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,32,1,128,1,float16,float16,0,0.08590559959411621
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,32,8,128,1,float16,float16,0,0.0987504005432129
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,32,32,128,1,float16,float16,0,0.08038079738616943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,32,1,128,1,float16,float16,0,0.05310879945755005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,32,2,128,1,float16,float16,0,0.05324959754943848
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,32,4,128,1,float16,float16,0,0.0573311984539032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,32,8,128,1,float16,float16,0,0.05600799918174744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,32,32,128,1,float16,float16,0,0.054504001140594484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,32,1,128,1,float16,float16,0,0.03782080113887787
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,32,2,128,1,float16,float16,0,0.03983359932899475
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,32,2,128,1,float16,float16,0,3.9053489685058596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,32,4,128,1,float16,float16,0,0.043198400735855104
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,32,8,128,1,float16,float16,0,0.04339680075645447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,32,1,128,1,float16,float16,0,1.2327919960021974
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,32,2,128,1,float16,float16,0,1.287054443359375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,32,8,128,1,float16,float16,0,1.0999327659606934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,32,4,128,1,float16,float16,0,1.3517519950866699
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,32,8,128,1,float16,float16,0,1.523465633392334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,32,1,128,1,float16,float16,0,0.6336912155151367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,32,2,128,1,float16,float16,0,0.655628776550293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,32,4,128,1,float16,float16,0,0.6943456172943115
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,32,8,128,1,float16,float16,0,0.16337440013885499
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,32,8,128,1,float16,float16,0,0.8002223968505859
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,32,4,128,1,float16,float16,0,0.09458879828453064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,32,32,128,1,float16,float16,0,1.4502047538757323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,32,1,128,1,float16,float16,0,0.3281327962875366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,32,32,128,1,float16,float16,0,0.7384175777435302
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,32,2,128,1,float16,float16,0,0.34059040546417235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,32,4,128,1,float16,float16,0,0.18927040100097656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,32,32,128,1,float16,float16,0,0.38334879875183103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,32,1,128,1,float16,float16,0,0.175163197517395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,32,4,128,1,float16,float16,0,0.35139520168304444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,32,1,128,1,float16,float16,0,0.09940959811210633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,32,2,128,1,float16,float16,0,0.10217599868774414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,32,4,128,1,float16,float16,0,0.10680160522460938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,32,8,128,1,float16,float16,0,0.11706559658050537
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,32,1,128,1,float16,float16,0,0.06238399744033814
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,32,32,128,1,float16,float16,0,0.12036319971084594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,32,2,128,1,float16,float16,0,0.06320000290870667
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,32,4,128,1,float16,float16,0,0.06559519767761231
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,32,32,128,1,float16,float16,0,0.06555200219154358
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,32,8,128,1,float16,float16,0,0.07131519913673401
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,32,1,128,1,float16,float16,0,0.03930079936981201
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,32,2,128,1,float16,float16,0,0.03899999856948853
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,32,4,128,1,float16,float16,0,0.039017599821090695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,32,8,128,1,float16,float16,0,0.04270560145378113
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,32,32,128,1,float16,float16,0,0.037028801441192624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,32,1,128,1,float16,float16,0,0.02773120105266571
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,32,2,128,1,float16,float16,0,0.02954559922218323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,32,4,128,1,float16,float16,0,0.028865599632263185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,32,32,128,1,float16,float16,0,0.025972801446914672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,32,8,128,1,float16,float16,0,0.02842240035533905
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,32,1,128,1,float16,float16,0,0.02282560020685196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,32,2,128,1,float16,float16,0,0.022732800245285033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,32,4,128,1,float16,float16,0,0.02316959947347641
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,32,8,128,1,float16,float16,0,0.023454399406909944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,32,1,128,1,float16,float16,0,0.4823584079742432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,32,2,128,1,float16,float16,0,0.5133984088897705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,32,4,128,1,float16,float16,0,0.5626336097717285
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,32,1,128,1,float16,float16,0,0.25666720867156984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,32,2,128,1,float16,float16,0,0.2710511922836304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,32,8,128,1,float16,float16,0,0.6658031940460205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,32,32,128,1,float16,float16,0,0.7057151794433594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,32,2,128,1,float16,float16,0,0.18065439462661742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,32,4,128,1,float16,float16,0,0.29736640453338625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,32,8,128,1,float16,float16,0,0.21512479782104493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,32,8,128,1,float16,float16,0,0.4035679817199707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,32,8,128,1,float16,float16,0,0.34684479236602783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,32,2,128,1,float16,float16,0,0.14564800262451172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,32,1,128,1,float16,float16,0,0.1394976019859314
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,32,32,128,1,float16,float16,0,0.20683200359344484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,32,32,128,1,float16,float16,0,0.3693264007568359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,32,4,128,1,float16,float16,0,0.15976799726486207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,32,8,128,1,float16,float16,0,0.1880959987640381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,32,1,128,1,float16,float16,0,0.08007519841194152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,32,2,128,1,float16,float16,0,0.08318719863891602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,32,8,128,1,float16,float16,0,0.10467679500579834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,32,4,128,1,float16,float16,0,0.09276639819145202
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,32,1,128,1,float16,float16,0,0.051767998933792116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,32,2,128,1,float16,float16,0,0.05288640260696411
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,32,8,128,1,float16,float16,0,0.06056960225105286
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,32,32,128,1,float16,float16,0,0.11267520189285278
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,32,32,128,1,float16,float16,0,0.05962560176849365
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,32,1,128,1,float16,float16,0,0.031064000725746155
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,32,2,128,1,float16,float16,0,0.0297791987657547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,32,4,128,1,float16,float16,0,0.029787200689315795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,32,32,128,1,float16,float16,0,0.02855519950389862
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,32,4,128,1,float16,float16,0,0.02189279943704605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,32,2,128,1,float16,float16,0,0.022998400032520294
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,32,8,128,1,float16,float16,0,0.023455999791622162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,32,32,128,1,float16,float16,0,0.02082560062408447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,32,1,128,1,float16,float16,0,0.0182096004486084
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,32,2,128,1,float16,float16,0,0.018313600122928618
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,32,4,128,1,float16,float16,0,0.018241600692272188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,32,8,128,1,float16,float16,0,0.018603199720382692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,32,32,128,1,float16,float16,0,0.01988160014152527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,32,1,128,1,float16,float16,0,0.017558400332927705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,32,2,128,1,float16,float16,0,0.017833599448204042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,32,4,128,1,float16,float16,0,0.01791200041770935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,32,8,128,1,float16,float16,0,0.01788319945335388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,32,2,128,1,float16,float16,0,0.25845439434051515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,32,1,128,1,float16,float16,0,0.24419839382171632
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,32,4,128,1,float16,float16,0,0.28714399337768554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,32,1,128,1,float16,float16,0,0.13358720541000366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,32,8,128,1,float16,float16,0,0.3423135995864868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,32,32,128,1,float16,float16,0,0.36983840465545653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,32,2,128,1,float16,float16,0,0.14108480215072633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,32,4,128,1,float16,float16,0,0.16014399528503417
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,32,8,128,1,float16,float16,0,0.18670079708099366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,32,1,128,1,float16,float16,0,0.07720479965209961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,32,32,128,1,float16,float16,0,0.19589279890060424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,32,2,128,1,float16,float16,0,0.08207839727401733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,32,4,128,1,float16,float16,0,0.08894720077514648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,32,8,128,1,float16,float16,0,0.10348320007324219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,32,32,128,1,float16,float16,0,0.10877920389175415
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,32,1,128,1,float16,float16,0,0.048990398645401
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,32,2,128,1,float16,float16,0,0.04997600018978119
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,32,8,128,1,float16,float16,0,0.05876799821853638
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,32,32,128,1,float16,float16,0,0.20037920475006105
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,32,32,128,1,float16,float16,0,0.05999839901924133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,32,1,128,1,float16,float16,0,0.02712000012397766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,32,4,128,1,float16,float16,0,0.05489439964294433
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,32,2,128,1,float16,float16,0,0.027025601267814635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,32,4,128,1,float16,float16,0,0.027399998903274537
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,32,8,128,1,float16,float16,0,0.029600000381469725
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,32,32,128,1,float16,float16,0,0.029076799750328064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,32,1,128,1,float16,float16,0,0.01871519982814789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,32,8,128,1,float16,float16,0,0.03169119954109192
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,32,1,128,1,float16,float16,0,0.022784000635147093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,32,2,128,1,float16,float16,0,0.0187376007437706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,32,8,128,1,float16,float16,0,0.019351999461650848
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,32,32,128,1,float16,float16,0,0.019566400349140166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,32,1,128,1,float16,float16,0,0.01576319932937622
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,32,2,128,1,float16,float16,0,0.01562879979610443
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,32,4,128,1,float16,float16,0,0.015868799388408662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,32,8,128,1,float16,float16,0,0.015971200168132783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,32,32,128,1,float16,float16,0,0.016414399445056915
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,32,4,128,1,float16,float16,0,0.01515360027551651
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,32,1,128,1,float16,float16,0,0.015289600193500518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,32,2,128,1,float16,float16,0,0.015121600031852723
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,32,32,128,1,float16,float16,0,0.015542399883270264
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,32,1,128,1,float16,float16,0,0.015035200119018554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,32,2,128,1,float16,float16,0,0.015175999701023101
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,32,4,128,1,float16,float16,0,0.015039999783039094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,32,8,128,1,float16,float16,0,0.015121600031852723
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,32,1,128,1,float16,float16,0,0.12820160388946533
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,32,2,128,1,float16,float16,0,0.13875360488891603
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,32,8,128,1,float16,float16,0,0.17539520263671876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,32,1,128,1,float16,float16,0,0.07728959918022156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,32,4,128,1,float16,float16,0,0.154203200340271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,32,32,128,1,float16,float16,0,0.24671199321746826
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,32,2,128,1,float16,float16,0,0.07996799945831298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,32,4,128,1,float16,float16,0,0.08755679726600647
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,32,8,128,1,float16,float16,0,0.10015679597854614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,32,1,128,1,float16,float16,0,0.04829919934272766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,32,32,128,1,float16,float16,0,0.13062880039215088
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,32,2,128,1,float16,float16,0,0.05050560235977173
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,32,4,128,1,float16,float16,0,0.05315840244293213
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,32,1,128,1,float16,float16,0,0.027551999688148497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,32,8,128,1,float16,float16,0,0.05888479948043823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,32,32,128,1,float16,float16,0,0.07173600196838378
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,32,2,128,1,float16,float16,0,0.027476799488067628
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,32,4,128,1,float16,float16,0,0.02792479991912842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,32,8,128,1,float16,float16,0,0.02951520085334778
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,32,32,128,1,float16,float16,0,0.036364799737930296
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,32,2,128,1,float16,float16,0,0.01838880032300949
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,32,4,128,1,float16,float16,0,0.018646399676799773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,32,8,128,1,float16,float16,0,0.018932799994945525
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,32,32,128,1,float16,float16,0,0.02519200146198273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,32,1,128,1,float16,float16,0,0.01528480052947998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,32,2,128,1,float16,float16,0,0.015542399883270264
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,32,4,128,1,float16,float16,0,0.015740799903869628
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,32,8,128,1,float16,float16,0,0.015857599675655365
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,32,32,128,1,float16,float16,0,0.018275199830532073
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,32,1,128,1,float16,float16,0,0.015092800557613372
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,32,2,128,1,float16,float16,0,0.015028800070285796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,32,4,128,1,float16,float16,0,0.015121600031852723
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,32,4,128,1,float16,float16,0,0.05248960256576538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,32,8,128,1,float16,float16,0,0.015404799580574035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,32,1,128,1,float16,float16,0,0.015086400508880615
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,32,2,128,1,float16,float16,0,0.014899200201034546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,32,4,128,1,float16,float16,0,0.015097600221633912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,32,8,128,1,float16,float16,0,0.015190400183200836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,32,32,128,1,float16,float16,0,0.01541759967803955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,32,1,128,1,float16,float16,0,0.014692799746990204
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,32,2,128,1,float16,float16,0,0.014824000000953675
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,32,4,128,1,float16,float16,0,0.014870400726795196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,32,8,128,1,float16,float16,0,0.014895999431610107
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,32,4,128,1,float16,float16,0,0.01902559995651245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,32,1,128,1,float16,float16,0,0.07769600152969361
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,32,2,128,1,float16,float16,0,0.08077920079231263
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,32,4,128,1,float16,float16,0,0.08875359892845154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,32,8,128,1,float16,float16,0,0.1240656018257141
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,32,1,128,1,float16,float16,0,0.05014240145683289
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,32,32,128,1,float16,float16,0,0.20051839351654052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,32,8,128,1,float16,float16,0,0.015299199521541596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,32,2,128,1,float16,float16,0,0.051367998123168945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,32,4,128,1,float16,float16,0,0.05408480167388916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,32,8,128,1,float16,float16,0,0.07062720060348511
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,32,1,128,1,float16,float16,0,0.028356799483299257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,32,2,128,1,float16,float16,0,0.028064000606536865
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,32,32,128,1,float16,float16,0,0.10715359449386597
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,32,4,128,1,float16,float16,0,0.029147198796272276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,32,8,128,1,float16,float16,0,0.036447998881340024
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,32,32,128,1,float16,float16,0,0.05382400155067444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,32,1,128,1,float16,float16,0,0.01876160055398941
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,32,2,128,1,float16,float16,0,0.01905920058488846
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,32,4,128,1,float16,float16,0,0.019383999705314636
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,32,8,128,1,float16,float16,0,0.02507199943065643
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,32,32,128,1,float16,float16,0,0.03274079859256744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,32,2,128,1,float16,float16,0,0.015643200278282164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,32,4,128,1,float16,float16,0,0.015905599296092986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,32,8,128,1,float16,float16,0,0.017579199373722078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,32,32,128,1,float16,float16,0,0.023848000168800353
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,32,1,128,1,float16,float16,0,0.015091200172901154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,32,2,128,1,float16,float16,0,0.015084800124168397
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,32,4,128,1,float16,float16,0,0.015332800149917603
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,32,8,128,1,float16,float16,0,0.015278400480747223
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,32,32,128,1,float16,float16,0,0.017481599748134614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,32,1,128,1,float16,float16,0,0.015139199793338776
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,32,2,128,1,float16,float16,0,0.0150736004114151
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,32,4,128,1,float16,float16,0,0.015030400454998016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,32,8,128,1,float16,float16,0,0.01467359960079193
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,32,32,128,1,float16,float16,0,0.015539200603961944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,32,1,128,1,float16,float16,0,0.014691199362277984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,32,2,128,1,float16,float16,0,0.014771200716495514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,32,4,128,1,float16,float16,0,0.014870400726795196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,32,8,128,1,float16,float16,0,0.014632000029087067
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,32,1,128,1,float16,float16,0,0.014812800288200378
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,32,32,128,1,float16,float16,0,0.015292799472808838
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,32,4,128,1,float16,float16,0,0.014870400726795196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,32,8,128,1,float16,float16,0,0.01435839980840683
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,32,1,128,1,float16,float16,0,0.018334400653839112
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,32,1,128,1,float16,float16,0,0.026654401421546937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,32,2,128,1,float16,float16,0,0.03396640121936798
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,32,4,128,1,float16,float16,0,0.05194240212440491
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,32,8,128,1,float16,float16,0,0.08893439769744874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,32,1,128,1,float16,float16,0,0.01866080015897751
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,32,2,128,1,float16,float16,0,0.02460640072822571
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,32,32,128,1,float16,float16,0,0.15515520572662353
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,32,4,128,1,float16,float16,0,0.03207519948482514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,32,8,128,1,float16,float16,0,0.049726399779319766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,32,1,128,1,float16,float16,0,0.015780800580978395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,32,2,128,1,float16,float16,0,0.017416000366210938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,32,32,128,1,float16,float16,0,0.08495360016822814
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,32,4,128,1,float16,float16,0,0.023388800024986268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,32,8,128,1,float16,float16,0,0.03099200129508972
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,32,1,128,1,float16,float16,0,0.015009599924087524
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,32,32,128,1,float16,float16,0,0.04799039959907532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,32,4,128,1,float16,float16,0,0.01693439930677414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,32,8,128,1,float16,float16,0,0.022974400222301482
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,32,32,128,1,float16,float16,0,0.03044799864292145
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,32,1,128,1,float16,float16,0,0.01422560065984726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,32,2,128,1,float16,float16,0,0.014508800208568573
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,32,4,128,1,float16,float16,0,0.014392000436782838
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,32,32,128,1,float16,float16,0,0.01557759940624237
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,32,8,128,1,float16,float16,0,0.016152000427246092
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,32,32,128,1,float16,float16,0,0.02280000001192093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,32,1,128,1,float16,float16,0,0.014057600498199463
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,32,2,128,1,float16,float16,0,0.014056000113487243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,32,8,128,1,float16,float16,0,0.01464959979057312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,32,32,128,1,float16,float16,0,0.016908800601959227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,32,1,128,1,float16,float16,0,0.01401440054178238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,32,2,128,1,float16,float16,0,0.014259199798107147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,32,8,128,1,float16,float16,0,0.014390400052070618
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,32,32,128,1,float16,float16,0,0.015364800393581391
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,32,1,128,1,float16,float16,0,0.014003199338912965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,32,2,128,1,float16,float16,0,0.014183999598026275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,32,4,128,1,float16,float16,0,0.013899199664592743
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,32,8,128,1,float16,float16,0,0.014363199472427368
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,32,1,128,1,float16,float16,0,0.013976000249385834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,32,2,128,1,float16,float16,0,0.014097599685192109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,32,4,128,1,float16,float16,0,0.013974399864673614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,32,8,128,1,float16,float16,0,0.014078399538993836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,32,1,128,1,float16,float16,0,0.015494400262832641
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,32,2,128,1,float16,float16,0,0.014878399670124054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,32,2,128,1,float16,float16,0,0.014937600493431092
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,32,4,128,1,float16,float16,0,0.014150400459766389
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,32,4,128,1,float16,float16,0,0.014023999869823455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,32,32,128,1,float16,float16,0,0.014972800016403198
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,24,1,128,1,float16,float16,0,31.969570922851563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,24,4,128,1,float16,float16,0,31.71137390136719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,24,2,128,1,float16,float16,0,32.27366943359375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,24,1,128,1,float16,float16,0,16.261990356445313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,24,1,128,1,float16,float16,0,62.47459106445312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,24,2,128,1,float16,float16,0,63.3209228515625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,24,4,128,1,float16,float16,0,63.659130859375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,24,8,128,1,float16,float16,0,64.7966796875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,24,8,128,1,float16,float16,0,32.518310546875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,24,2,128,1,float16,float16,0,16.297166442871095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,24,1,128,1,float16,float16,0,8.207577514648438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,24,2,128,1,float16,float16,0,8.151742553710937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,24,4,128,1,float16,float16,0,15.98151092529297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,24,8,128,1,float16,float16,0,16.353298950195313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,24,4,128,1,float16,float16,0,8.29556655883789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,24,8,128,1,float16,float16,0,8.352190399169922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,24,2,128,1,float16,float16,0,35.44380187988281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,24,1,128,1,float16,float16,0,36.42225341796875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,24,4,128,1,float16,float16,0,36.20189514160156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,24,8,128,1,float16,float16,0,36.98137512207031
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,24,1,128,1,float16,float16,0,17.780226135253905
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,24,2,128,1,float16,float16,0,18.349131774902343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,24,4,128,1,float16,float16,0,18.2279541015625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,24,8,128,1,float16,float16,0,18.514558410644533
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,24,1,128,1,float16,float16,0,9.203321838378907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,24,2,128,1,float16,float16,0,9.131281280517578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,24,4,128,1,float16,float16,0,9.258358764648438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,24,8,128,1,float16,float16,0,9.535765075683594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,24,1,128,1,float16,float16,0,4.66984977722168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,24,2,128,1,float16,float16,0,4.685111999511719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,24,4,128,1,float16,float16,0,4.774603271484375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,24,8,128,1,float16,float16,0,4.873406219482422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,24,1,128,1,float16,float16,0,25.29649963378906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,24,2,128,1,float16,float16,0,25.514933776855468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,24,4,128,1,float16,float16,0,25.582386779785157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,24,8,128,1,float16,float16,0,26.028274536132812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,24,1,128,1,float16,float16,0,12.826870727539063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,24,2,128,1,float16,float16,0,12.8286376953125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,24,24,128,1,float16,float16,0,4.891505432128906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,24,4,128,1,float16,float16,0,12.980314636230469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,24,24,128,1,float16,float16,0,8.559703826904297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,24,24,128,1,float16,float16,0,9.376108551025391
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,24,24,128,1,float16,float16,0,6.940895843505859
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,24,24,128,1,float16,float16,0,13.501821899414063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,24,1,128,1,float16,float16,0,6.291308975219726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,24,8,128,1,float16,float16,0,12.856210327148437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,24,24,128,1,float16,float16,0,16.70434112548828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,24,2,128,1,float16,float16,0,6.2583168029785154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,24,4,128,1,float16,float16,0,6.564044952392578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,24,24,128,1,float16,float16,0,3.496535873413086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,24,24,128,1,float16,float16,0,19.060406494140626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,24,1,128,1,float16,float16,0,3.261374282836914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,24,8,128,1,float16,float16,0,6.571025848388672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,24,2,128,1,float16,float16,0,3.2289710998535157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,24,4,128,1,float16,float16,0,3.251232147216797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,24,8,128,1,float16,float16,0,3.454417419433594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,24,24,128,1,float16,float16,0,33.21089172363281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,24,1,128,1,float16,float16,0,15.918196105957032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,24,2,128,1,float16,float16,0,16.459274291992188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,24,24,128,1,float16,float16,0,17.290614318847656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,24,1,128,1,float16,float16,0,7.989036560058594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,24,4,128,1,float16,float16,0,16.229765319824217
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,24,24,128,1,float16,float16,0,8.955757141113281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,24,1,128,1,float16,float16,0,32.06761169433594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,24,2,128,1,float16,float16,0,32.490386962890625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,24,4,128,1,float16,float16,0,32.4276123046875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,24,8,128,1,float16,float16,0,16.371051025390624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,24,8,128,1,float16,float16,0,33.384375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,24,2,128,1,float16,float16,0,8.097100830078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,24,24,128,1,float16,float16,0,4.471084976196289
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,24,4,128,1,float16,float16,0,8.164147186279298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,24,8,128,1,float16,float16,0,8.506438446044921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,24,1,128,1,float16,float16,0,4.092174530029297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,24,2,128,1,float16,float16,0,4.164678573608398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,24,24,128,1,float16,float16,0,2.335139274597168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,24,1,128,1,float16,float16,0,2.135628890991211
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,24,4,128,1,float16,float16,0,4.132174301147461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,24,2,128,1,float16,float16,0,2.1294736862182617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,24,8,128,1,float16,float16,0,4.304792022705078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,24,4,128,1,float16,float16,0,2.243084716796875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,24,8,128,1,float16,float16,0,2.250667190551758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,24,1,128,1,float16,float16,0,9.253902435302734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,24,2,128,1,float16,float16,0,9.392593383789062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,24,24,128,1,float16,float16,0,10.05985107421875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,24,4,128,1,float16,float16,0,9.360926055908203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,24,1,128,1,float16,float16,0,4.591886520385742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,24,2,128,1,float16,float16,0,4.628944015502929
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,24,24,128,1,float16,float16,0,5.188030242919922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,24,1,128,1,float16,float16,0,18.265440368652342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,24,2,128,1,float16,float16,0,18.134481811523436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,24,4,128,1,float16,float16,0,18.526797485351562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,24,8,128,1,float16,float16,0,18.612773132324218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,24,24,128,1,float16,float16,0,2.5968128204345704
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,24,1,128,1,float16,float16,0,2.330939292907715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,24,2,128,1,float16,float16,0,2.3528112411499023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,24,4,128,1,float16,float16,0,2.442265510559082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,24,1,128,1,float16,float16,0,1.2239839553833007
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,24,24,128,1,float16,float16,0,1.4154080390930175
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,24,8,128,1,float16,float16,0,2.453041648864746
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,24,2,128,1,float16,float16,0,1.2625120162963868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,24,4,128,1,float16,float16,0,4.649835205078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,24,8,128,1,float16,float16,0,4.768270492553711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,24,4,128,1,float16,float16,0,1.3170736312866211
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,24,8,128,1,float16,float16,0,1.3275055885314941
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,24,8,128,1,float16,float16,0,9.548681640625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,24,1,128,1,float16,float16,0,8.300940704345702
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,24,2,128,1,float16,float16,0,8.326096343994141
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,24,24,128,1,float16,float16,0,9.528243255615234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,24,4,128,1,float16,float16,0,8.494161224365234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,24,1,128,1,float16,float16,0,4.172391891479492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,24,24,128,1,float16,float16,0,4.7552238464355465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,24,2,128,1,float16,float16,0,4.223809432983399
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,24,2,128,1,float16,float16,0,16.418852233886717
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,24,1,128,1,float16,float16,0,16.84205322265625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,24,4,128,1,float16,float16,0,16.75343017578125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,24,8,128,1,float16,float16,0,17.157769775390626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,24,8,128,1,float16,float16,0,8.727430725097657
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,24,24,128,1,float16,float16,0,2.3839120864868164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,24,4,128,1,float16,float16,0,4.260323333740234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,24,1,128,1,float16,float16,0,2.1531583786010744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,24,2,128,1,float16,float16,0,2.129599952697754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,24,8,128,1,float16,float16,0,4.365439987182617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,24,4,128,1,float16,float16,0,2.161310386657715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,24,1,128,1,float16,float16,0,1.1246095657348634
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,24,24,128,1,float16,float16,0,1.250767993927002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,24,2,128,1,float16,float16,0,1.158465576171875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,24,4,128,1,float16,float16,0,1.1595104217529297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,24,8,128,1,float16,float16,0,2.2267520904541014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,24,8,128,1,float16,float16,0,1.2264047622680665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,24,24,128,1,float16,float16,0,0.709012794494629
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,24,1,128,1,float16,float16,0,0.6029759883880615
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,24,4,128,1,float16,float16,0,0.6639791965484619
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,24,2,128,1,float16,float16,0,0.6146160125732422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,24,8,128,1,float16,float16,0,0.6585343837738037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,24,1,128,1,float16,float16,0,4.681134414672852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,24,2,128,1,float16,float16,0,4.728507232666016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,24,24,128,1,float16,float16,0,5.680654525756836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,24,1,128,1,float16,float16,0,2.423259162902832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,24,24,128,1,float16,float16,0,2.7672943115234374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,24,1,128,1,float16,float16,0,9.472879791259766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,24,2,128,1,float16,float16,0,9.618077087402344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,24,4,128,1,float16,float16,0,9.670196533203125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,24,8,128,1,float16,float16,0,9.973375701904297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,24,4,128,1,float16,float16,0,4.772711944580078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,24,8,128,1,float16,float16,0,4.957326507568359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,24,1,128,1,float16,float16,0,1.239475154876709
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,24,24,128,1,float16,float16,0,1.466487979888916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,24,2,128,1,float16,float16,0,2.4608463287353515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,24,2,128,1,float16,float16,0,1.2715999603271484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,24,4,128,1,float16,float16,0,2.445902442932129
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,24,8,128,1,float16,float16,0,2.5299728393554686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,24,8,128,1,float16,float16,0,1.3115519523620605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,24,1,128,1,float16,float16,0,0.6645904064178467
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,24,24,128,1,float16,float16,0,0.7715904235839843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,24,4,128,1,float16,float16,0,0.7007999897003174
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,24,1,128,1,float16,float16,0,0.3789920091629028
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,24,24,128,1,float16,float16,0,0.4647552013397217
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,24,2,128,1,float16,float16,0,0.38022239208221437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,24,8,128,1,float16,float16,0,0.7385087966918945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,24,4,128,1,float16,float16,0,0.40024480819702146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,24,8,128,1,float16,float16,0,0.41287841796875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,24,4,128,1,float16,float16,0,1.2714447975158691
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,24,2,128,1,float16,float16,0,0.7087584018707276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,24,1,128,1,float16,float16,0,4.308768081665039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,24,24,128,1,float16,float16,0,5.385497665405273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,24,2,128,1,float16,float16,0,4.291912078857422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,24,4,128,1,float16,float16,0,4.519168090820313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,24,1,128,1,float16,float16,0,8.678939056396484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,24,1,128,1,float16,float16,0,2.1608848571777344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,24,2,128,1,float16,float16,0,8.838432312011719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,24,2,128,1,float16,float16,0,2.23449592590332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,24,4,128,1,float16,float16,0,9.079019165039062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,24,8,128,1,float16,float16,0,4.706014251708984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,24,8,128,1,float16,float16,0,9.439408111572266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,24,1,128,1,float16,float16,0,1.1509455680847167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,24,4,128,1,float16,float16,0,2.247452735900879
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,24,24,128,1,float16,float16,0,1.4114144325256348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,24,2,128,1,float16,float16,0,1.1576671600341797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,24,8,128,1,float16,float16,0,2.3131776809692384
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,24,4,128,1,float16,float16,0,1.1738495826721191
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,24,8,128,1,float16,float16,0,1.23919677734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,24,1,128,1,float16,float16,0,0.6136591911315918
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,24,2,128,1,float16,float16,0,0.6036064147949218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,24,24,128,1,float16,float16,0,0.7443664073944092
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,24,24,128,1,float16,float16,0,0.4109360218048096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,24,1,128,1,float16,float16,0,0.33393120765686035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,24,4,128,1,float16,float16,0,0.6329360008239746
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,24,8,128,1,float16,float16,0,0.64759840965271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,24,2,128,1,float16,float16,0,0.3485935926437378
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,24,4,128,1,float16,float16,0,0.34530398845672605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,24,8,128,1,float16,float16,0,0.36321918964385985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,24,24,128,1,float16,float16,0,0.24076321125030517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,24,2,128,1,float16,float16,0,0.2141871929168701
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,24,1,128,1,float16,float16,0,0.20469920635223388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,24,4,128,1,float16,float16,0,0.20995841026306153
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,24,8,128,1,float16,float16,0,0.21580960750579833
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,24,24,128,1,float16,float16,0,2.7408912658691404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,24,1,128,1,float16,float16,0,2.466540718078613
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,24,2,128,1,float16,float16,0,2.5004255294799806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,24,4,128,1,float16,float16,0,2.559004783630371
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,24,24,128,1,float16,float16,0,3.3357025146484376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,24,1,128,1,float16,float16,0,1.2906255722045898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,24,24,128,1,float16,float16,0,1.6829696655273438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,24,1,128,1,float16,float16,0,5.08851203918457
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,24,2,128,1,float16,float16,0,5.20816650390625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,24,4,128,1,float16,float16,0,5.261939239501953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,24,2,128,1,float16,float16,0,1.318171215057373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,24,8,128,1,float16,float16,0,5.623700714111328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,24,8,128,1,float16,float16,0,2.7508159637451173
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,24,24,128,1,float16,float16,0,0.8824272155761719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,24,1,128,1,float16,float16,0,0.7000832080841064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,24,4,128,1,float16,float16,0,1.3123392105102538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,24,2,128,1,float16,float16,0,0.7048975944519043
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,24,4,128,1,float16,float16,0,0.6931183815002442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,24,8,128,1,float16,float16,0,1.3911552429199219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,24,24,128,1,float16,float16,0,0.4751743793487549
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,24,1,128,1,float16,float16,0,0.3761679887771606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,24,8,128,1,float16,float16,0,0.7566944122314453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,24,2,128,1,float16,float16,0,0.3812463998794556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,24,4,128,1,float16,float16,0,0.3893167972564697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,24,8,128,1,float16,float16,0,0.3921999931335449
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,24,24,128,1,float16,float16,0,0.2737584114074707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,24,2,128,1,float16,float16,0,0.2212656021118164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,24,1,128,1,float16,float16,0,0.2058624029159546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,24,4,128,1,float16,float16,0,0.22973279953002929
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,24,1,128,1,float16,float16,0,0.13793760538101196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,24,8,128,1,float16,float16,0,0.23332159519195556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,24,2,128,1,float16,float16,0,0.1444048047065735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,24,4,128,1,float16,float16,0,0.13560160398483276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,24,8,128,1,float16,float16,0,0.13842400312423705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,24,1,128,1,float16,float16,0,2.3234527587890623
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,24,24,128,1,float16,float16,0,0.16813440322875978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,24,24,128,1,float16,float16,0,3.5044078826904297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,24,1,128,1,float16,float16,0,4.844585418701172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,24,2,128,1,float16,float16,0,2.404563140869141
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,24,4,128,1,float16,float16,0,5.0643054962158205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,24,8,128,1,float16,float16,0,5.444472122192383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,24,1,128,1,float16,float16,0,1.2026800155639648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,24,2,128,1,float16,float16,0,1.2405247688293457
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,24,24,128,1,float16,float16,0,1.7640239715576171
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,24,4,128,1,float16,float16,0,1.2865504264831542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,24,8,128,1,float16,float16,0,2.7075216293334963
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,24,4,128,1,float16,float16,0,2.3978239059448243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,24,1,128,1,float16,float16,0,0.6105103969573975
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,24,2,128,1,float16,float16,0,0.6375679969787598
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,24,4,128,1,float16,float16,0,0.6590400218963623
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,24,24,128,1,float16,float16,0,0.8904047966003418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,24,8,128,1,float16,float16,0,1.348464012145996
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,24,24,128,1,float16,float16,0,0.49307999610900877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,24,1,128,1,float16,float16,0,0.32832319736480714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,24,2,128,1,float16,float16,0,0.34506878852844236
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,24,8,128,1,float16,float16,0,0.7417920112609864
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,24,4,128,1,float16,float16,0,0.34940481185913086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,24,8,128,1,float16,float16,0,0.3775183916091919
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,24,24,128,1,float16,float16,0,0.26791679859161377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,24,2,128,1,float16,float16,0,0.1917296051979065
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,24,1,128,1,float16,float16,0,0.1973296046257019
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,24,4,128,1,float16,float16,0,0.20101919174194335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,24,24,128,1,float16,float16,0,0.15331200361251832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,24,8,128,1,float16,float16,0,0.22094080448150635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,24,1,128,1,float16,float16,0,0.11664479970932007
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,24,2,128,1,float16,float16,0,0.12038559913635254
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,24,4,128,1,float16,float16,0,0.1181920051574707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,24,24,128,1,float16,float16,0,0.08625440001487732
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,24,1,128,1,float16,float16,0,0.06642240285873413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,24,8,128,1,float16,float16,0,0.13518879413604737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,24,2,128,1,float16,float16,0,0.07979199886322022
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,24,8,128,1,float16,float16,0,0.07147039771080017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,24,1,128,1,float16,float16,0,1.400486373901367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,24,2,128,1,float16,float16,0,4.9213920593261715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,24,24,128,1,float16,float16,0,2.5012399673461916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,24,1,128,1,float16,float16,0,2.814169692993164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,24,2,128,1,float16,float16,0,2.9322847366333007
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,24,4,128,1,float16,float16,0,3.0772703170776365
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,24,2,128,1,float16,float16,0,1.4594767570495606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,24,8,128,1,float16,float16,0,3.505684661865234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,24,1,128,1,float16,float16,0,0.7522255897521972
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,24,2,128,1,float16,float16,0,0.7637904167175293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,24,4,128,1,float16,float16,0,1.5088512420654296
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,24,4,128,1,float16,float16,0,0.7872176170349121
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,24,24,128,1,float16,float16,0,1.293006420135498
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,24,8,128,1,float16,float16,0,1.7632959365844727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,24,8,128,1,float16,float16,0,0.9181072235107421
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,24,1,128,1,float16,float16,0,0.378603196144104
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,24,2,128,1,float16,float16,0,0.39417281150817873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,24,4,128,1,float16,float16,0,0.40393919944763185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,24,24,128,1,float16,float16,0,0.6720911979675293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,24,1,128,1,float16,float16,0,0.2138063907623291
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,24,24,128,1,float16,float16,0,0.36417279243469236
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,24,2,128,1,float16,float16,0,0.20815680027008057
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,24,1,128,1,float16,float16,0,0.1202720046043396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,24,2,128,1,float16,float16,0,0.1245568037033081
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,24,8,128,1,float16,float16,0,0.24493920803070068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,24,24,128,1,float16,float16,0,0.19556159973144532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,24,4,128,1,float16,float16,0,0.12485760450363159
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,24,4,128,1,float16,float16,0,0.07863039970397949
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,24,24,128,1,float16,float16,0,0.10790560245513917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,24,8,128,1,float16,float16,0,0.13984479904174804
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,24,8,128,1,float16,float16,0,0.08480160236358643
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,24,24,128,1,float16,float16,0,0.0667519986629486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,24,1,128,1,float16,float16,0,0.046900799870491026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,24,2,128,1,float16,float16,0,0.04503360092639923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,24,4,128,1,float16,float16,0,0.04294399917125702
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,24,8,128,1,float16,float16,0,0.050697600841522215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,24,24,128,1,float16,float16,0,0.03729119896888733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,24,1,128,1,float16,float16,0,0.03367680013179779
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,24,2,128,1,float16,float16,0,0.03345920145511627
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,24,4,128,1,float16,float16,0,0.03384479880332947
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,24,4,128,1,float16,float16,0,0.07499679923057556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,24,8,128,1,float16,float16,0,0.03483360111713409
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,24,1,128,1,float16,float16,0,0.945622444152832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,24,2,128,1,float16,float16,0,0.9857888221740723
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,24,4,128,1,float16,float16,0,1.060689640045166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,24,8,128,1,float16,float16,0,0.4647520065307617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,24,1,128,1,float16,float16,0,0.48001441955566404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,24,4,128,1,float16,float16,0,0.21981279850006102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,24,8,128,1,float16,float16,0,1.294262409210205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,24,2,128,1,float16,float16,0,0.5057055950164795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,24,24,128,1,float16,float16,0,1.099499225616455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,24,2,128,1,float16,float16,0,0.07678400278091431
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,24,4,128,1,float16,float16,0,0.5427375793457031
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,24,1,128,1,float16,float16,0,0.25235040187835694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,24,2,128,1,float16,float16,0,0.26999359130859374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,24,8,128,1,float16,float16,0,0.6579328060150147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,24,4,128,1,float16,float16,0,0.28512959480285643
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,24,24,128,1,float16,float16,0,0.5757616043090821
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,24,8,128,1,float16,float16,0,0.34326560497283937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,24,1,128,1,float16,float16,0,0.14096800088882447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,24,2,128,1,float16,float16,0,0.14501919746398925
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,24,24,128,1,float16,float16,0,0.2953727960586548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,24,4,128,1,float16,float16,0,0.1549728035926819
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,24,8,128,1,float16,float16,0,0.1904271960258484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,24,24,128,1,float16,float16,0,0.1644927978515625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,24,8,128,1,float16,float16,0,0.10262720584869385
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,24,4,128,1,float16,float16,0,0.08970080018043518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,24,24,128,1,float16,float16,0,0.09335839748382568
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,24,2,128,1,float16,float16,0,0.05248960256576538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,24,1,128,1,float16,float16,0,0.05132160186767578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,24,4,128,1,float16,float16,0,0.05613279938697815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,24,8,128,1,float16,float16,0,0.06352319717407226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,24,2,128,1,float16,float16,0,0.02858240008354187
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,24,1,128,1,float16,float16,0,0.030236798524856567
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,24,4,128,1,float16,float16,0,0.03274079859256744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,24,8,128,1,float16,float16,0,0.033907198905944826
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,24,24,128,1,float16,float16,0,0.027372801303863527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,24,1,128,1,float16,float16,0,0.023526400327682495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,24,2,128,1,float16,float16,0,0.023716799914836884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,24,4,128,1,float16,float16,0,0.023689599335193635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,24,8,128,1,float16,float16,0,0.025360000133514405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,24,24,128,1,float16,float16,0,0.025443199276924133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,24,1,128,1,float16,float16,0,0.022204799950122832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,24,2,128,1,float16,float16,0,0.02311040014028549
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,24,4,128,1,float16,float16,0,0.023014399409294128
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,24,8,128,1,float16,float16,0,0.02375199943780899
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,24,1,128,1,float16,float16,0,0.07346720099449158
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,24,1,128,1,float16,float16,0,0.3791215896606445
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,24,2,128,1,float16,float16,0,0.40558562278747556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,24,4,128,1,float16,float16,0,0.4498623847961426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,24,8,128,1,float16,float16,0,0.5729936122894287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,24,1,128,1,float16,float16,0,0.20000801086425782
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,24,1,128,1,float16,float16,0,0.08203039765357971
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,24,2,128,1,float16,float16,0,0.21717441082000732
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,24,24,128,1,float16,float16,0,0.5406799793243409
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,24,24,128,1,float16,float16,0,0.049686399102210996
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,24,4,128,1,float16,float16,0,0.24340639114379883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,24,1,128,1,float16,float16,0,0.1110543966293335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,24,24,128,1,float16,float16,0,0.2843600034713745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,24,8,128,1,float16,float16,0,0.30241439342498777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,24,2,128,1,float16,float16,0,0.11900320053100585
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,24,4,128,1,float16,float16,0,0.13139679431915283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,24,24,128,1,float16,float16,0,0.15566400289535523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,24,8,128,1,float16,float16,0,0.1678815960884094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,24,1,128,1,float16,float16,0,0.0669920027256012
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,24,4,128,1,float16,float16,0,0.07608000040054322
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,24,2,128,1,float16,float16,0,0.06971200108528137
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,24,24,128,1,float16,float16,0,0.08196319937705994
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,24,2,128,1,float16,float16,0,0.04007999897003174
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,24,4,128,1,float16,float16,0,0.04414879977703094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,24,1,128,1,float16,float16,0,0.03811039924621582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,24,1,128,1,float16,float16,0,0.021960000693798064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,24,24,128,1,float16,float16,0,0.04167360067367554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,24,4,128,1,float16,float16,0,0.02407200038433075
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,24,8,128,1,float16,float16,0,0.026262399554252625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,24,2,128,1,float16,float16,0,0.023937599360942842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,24,2,128,1,float16,float16,0,0.018812799453735353
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,24,4,128,1,float16,float16,0,0.019222399592399596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,24,8,128,1,float16,float16,0,0.01993120014667511
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,24,24,128,1,float16,float16,0,0.02022400051355362
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,24,1,128,1,float16,float16,0,0.018089599907398224
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,24,2,128,1,float16,float16,0,0.01780479997396469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,24,4,128,1,float16,float16,0,0.01807360053062439
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,24,8,128,1,float16,float16,0,0.018929600715637207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,24,24,128,1,float16,float16,0,0.019256000220775605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,24,1,128,1,float16,float16,0,0.017524799704551695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,24,4,128,1,float16,float16,0,0.017590400576591492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,24,2,128,1,float16,float16,0,0.01785759925842285
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,24,8,128,1,float16,float16,0,0.01836480051279068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,24,2,128,1,float16,float16,0,0.08404639959335328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,24,8,128,1,float16,float16,0,0.09363679885864258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,24,1,128,1,float16,float16,0,0.1900272011756897
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,24,2,128,1,float16,float16,0,0.20500640869140624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,24,8,128,1,float16,float16,0,0.05230879783630371
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,24,1,128,1,float16,float16,0,0.10578880310058594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,24,8,128,1,float16,float16,0,0.3025199890136719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,24,4,128,1,float16,float16,0,0.23402559757232666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,24,2,128,1,float16,float16,0,0.11395679712295533
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,24,4,128,1,float16,float16,0,0.12975360155105592
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,24,24,128,1,float16,float16,0,0.2830879926681519
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,24,1,128,1,float16,float16,0,0.018531200289726258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,24,24,128,1,float16,float16,0,0.02252960056066513
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,24,8,128,1,float16,float16,0,0.16343519687652588
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,24,2,128,1,float16,float16,0,0.06668480038642884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,24,24,128,1,float16,float16,0,0.15239039659500123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,24,24,128,1,float16,float16,0,0.08434079885482788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,24,1,128,1,float16,float16,0,0.03291999995708465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,24,4,128,1,float16,float16,0,0.04189119935035705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,24,8,128,1,float16,float16,0,0.053193598985672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,24,1,128,1,float16,float16,0,0.0200080007314682
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,24,2,128,1,float16,float16,0,0.020187200605869295
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,24,24,128,1,float16,float16,0,0.04374400079250336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,24,4,128,1,float16,float16,0,0.020422400534152986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,24,24,128,1,float16,float16,0,0.021316799521446227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,24,1,128,1,float16,float16,0,0.016484799981117248
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,24,2,128,1,float16,float16,0,0.01661760061979294
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,24,4,128,1,float16,float16,0,0.016808000206947327
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,24,8,128,1,float16,float16,0,0.019169600307941438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,24,24,128,1,float16,float16,0,0.016915200650691985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,24,1,128,1,float16,float16,0,0.015321600437164306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,24,2,128,1,float16,float16,0,0.015292799472808838
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,24,4,128,1,float16,float16,0,0.015612800419330598
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,24,8,128,1,float16,float16,0,0.015703999996185304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,24,24,128,1,float16,float16,0,0.01587039977312088
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,24,1,128,1,float16,float16,0,0.014936000108718872
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,24,2,128,1,float16,float16,0,0.015174399316310882
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,24,4,128,1,float16,float16,0,0.014982399344444276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,24,8,128,1,float16,float16,0,0.01547040045261383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,24,24,128,1,float16,float16,0,0.01520639955997467
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,24,1,128,1,float16,float16,0,0.014825600385665893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,24,2,128,1,float16,float16,0,0.014897599816322327
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,24,4,128,1,float16,float16,0,0.014800000190734863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,24,8,128,1,float16,float16,0,0.015009599924087524
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,24,1,128,1,float16,float16,0,0.10445599555969239
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,24,2,128,1,float16,float16,0,0.11383680105209351
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,24,4,128,1,float16,float16,0,0.13904320001602172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,24,8,128,1,float16,float16,0,0.1576607942581177
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,24,1,128,1,float16,float16,0,0.06273760199546814
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,24,24,128,1,float16,float16,0,0.18997119665145873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,24,2,128,1,float16,float16,0,0.06709120273590088
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,24,4,128,1,float16,float16,0,0.07880319952964783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,24,8,128,1,float16,float16,0,0.08981919884681702
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,24,24,128,1,float16,float16,0,0.10140479803085327
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,24,1,128,1,float16,float16,0,0.06352319717407226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,24,4,128,1,float16,float16,0,0.07437919974327087
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,24,1,128,1,float16,float16,0,0.03346399962902069
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,24,8,128,1,float16,float16,0,0.09355520009994507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,24,2,128,1,float16,float16,0,0.03645919859409332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,24,8,128,1,float16,float16,0,0.05302559733390808
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,24,24,128,1,float16,float16,0,0.05743520259857178
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,24,1,128,1,float16,float16,0,0.020414400100708007
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,24,2,128,1,float16,float16,0,0.0208064004778862
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,24,4,128,1,float16,float16,0,0.026604801416397095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,24,8,128,1,float16,float16,0,0.027377599477767946
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,24,1,128,1,float16,float16,0,0.01636800020933151
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,24,24,128,1,float16,float16,0,0.02860639989376068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,24,2,128,1,float16,float16,0,0.016523200273513793
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,24,4,128,1,float16,float16,0,0.01860000044107437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,24,8,128,1,float16,float16,0,0.018532800674438476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,24,24,128,1,float16,float16,0,0.018747200071811677
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,24,1,128,1,float16,float16,0,0.015275199711322785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,24,2,128,1,float16,float16,0,0.015352000296115876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,24,4,128,1,float16,float16,0,0.015598399937152863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,24,8,128,1,float16,float16,0,0.015603199601173401
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,24,24,128,1,float16,float16,0,0.01610880047082901
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,24,1,128,1,float16,float16,0,0.015048000216484069
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,24,2,128,1,float16,float16,0,0.015097600221633912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,24,8,128,1,float16,float16,0,0.015137599408626556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,24,4,128,1,float16,float16,0,0.015035200119018554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,24,24,128,1,float16,float16,0,0.015326400101184846
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,24,1,128,1,float16,float16,0,0.014902399480342865
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,24,2,128,1,float16,float16,0,0.015012800693511963
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,24,4,128,1,float16,float16,0,0.014883199334144592
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,24,8,128,1,float16,float16,0,0.014803199470043183
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,24,24,128,1,float16,float16,0,0.01512639969587326
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,24,1,128,1,float16,float16,0,0.014951999485492706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,24,2,128,1,float16,float16,0,0.014876799285411834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,24,4,128,1,float16,float16,0,0.014753599464893342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,24,8,128,1,float16,float16,0,0.014732800424098969
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,24,1,128,1,float16,float16,0,0.06563519835472106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,24,2,128,1,float16,float16,0,0.07488960027694702
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,24,8,128,1,float16,float16,0,0.11995999813079834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,24,1,128,1,float16,float16,0,0.034123200178146365
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,24,24,128,1,float16,float16,0,0.15598560571670533
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,24,2,128,1,float16,float16,0,0.043572801351547244
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,24,4,128,1,float16,float16,0,0.04791040122509003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,24,8,128,1,float16,float16,0,0.06711680293083191
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,24,24,128,1,float16,float16,0,0.08328480124473572
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,24,1,128,1,float16,float16,0,0.020948800444602966
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,24,2,128,1,float16,float16,0,0.026712000370025635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,24,4,128,1,float16,float16,0,0.02667680084705353
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,24,8,128,1,float16,float16,0,0.03519999980926514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,24,24,128,1,float16,float16,0,0.043609601259231565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,24,1,128,1,float16,float16,0,0.01675360053777695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,24,2,128,1,float16,float16,0,0.018755200505256652
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,24,4,128,1,float16,float16,0,0.018801599740982056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,24,8,128,1,float16,float16,0,0.024563199281692503
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,24,24,128,1,float16,float16,0,0.026601600646972656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,24,1,128,1,float16,float16,0,0.015175999701023101
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,24,2,128,1,float16,float16,0,0.015134400129318238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,24,4,128,1,float16,float16,0,0.015396800637245179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,24,8,128,1,float16,float16,0,0.017377600073814392
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,24,1,128,1,float16,float16,0,0.015068799257278442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,24,24,128,1,float16,float16,0,0.017745600640773775
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,24,2,128,1,float16,float16,0,0.014983999729156493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,24,4,128,1,float16,float16,0,0.014975999295711518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,24,8,128,1,float16,float16,0,0.015043200552463531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,24,24,128,1,float16,float16,0,0.01547040045261383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,24,1,128,1,float16,float16,0,0.014767999947071075
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,24,2,128,1,float16,float16,0,0.014839999377727509
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,24,4,128,1,float16,float16,0,0.04747999906539917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,24,2,128,1,float16,float16,0,0.03617120087146759
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,24,4,128,1,float16,float16,0,0.01478240042924881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,24,8,128,1,float16,float16,0,0.015406399965286255
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,24,1,128,1,float16,float16,0,0.014910399913787842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,24,4,128,1,float16,float16,0,0.014620800316333771
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,24,8,128,1,float16,float16,0,0.014531199634075165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,24,24,128,1,float16,float16,0,0.014982399344444276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,24,1,128,1,float16,float16,0,0.014844800531864166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,24,2,128,1,float16,float16,0,0.014798399806022645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,24,4,128,1,float16,float16,0,0.014655999839305878
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,24,8,128,1,float16,float16,0,0.014451199769973755
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,24,8,128,1,float16,float16,0,0.0270687997341156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,24,1,128,1,float16,float16,0,0.026587200164794923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,24,2,128,1,float16,float16,0,0.03381119966506958
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,24,4,128,1,float16,float16,0,0.051815998554229734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,24,8,128,1,float16,float16,0,0.08933280110359192
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,24,1,128,1,float16,float16,0,0.01860480010509491
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,24,24,128,1,float16,float16,0,0.1209231972694397
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,24,2,128,1,float16,float16,0,0.024668799340724946
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,24,4,128,1,float16,float16,0,0.03215999901294708
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,24,8,128,1,float16,float16,0,0.04984320104122162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,24,1,128,1,float16,float16,0,0.015582400560379028
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,24,24,128,1,float16,float16,0,0.06769919991493226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,24,4,128,1,float16,float16,0,0.023529599606990814
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,24,8,128,1,float16,float16,0,0.03135200142860413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,24,24,128,1,float16,float16,0,0.04003680050373078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,24,1,128,1,float16,float16,0,0.01512639969587326
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,24,4,128,1,float16,float16,0,0.016726399958133697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,24,8,128,1,float16,float16,0,0.022974400222301482
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,24,24,128,1,float16,float16,0,0.024771200120449068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,24,1,128,1,float16,float16,0,0.014417600631713868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,24,2,128,1,float16,float16,0,0.01417279988527298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,24,8,128,1,float16,float16,0,0.016180799901485445
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,24,24,128,1,float16,float16,0,0.01687040030956268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,24,1,128,1,float16,float16,0,0.013920000195503235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,24,2,128,1,float16,float16,0,0.013961599767208099
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,24,4,128,1,float16,float16,0,0.014270399510860444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,24,8,128,1,float16,float16,0,0.014412799477577209
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,24,24,128,1,float16,float16,0,0.015171200037002563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,24,1,128,1,float16,float16,0,0.013862399756908417
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,24,2,128,1,float16,float16,0,0.013990400731563568
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,24,4,128,1,float16,float16,0,0.01403679996728897
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,24,8,128,1,float16,float16,0,0.014206400513648987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,24,24,128,1,float16,float16,0,0.014852799475193024
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,24,1,128,1,float16,float16,0,0.013780799508094788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,24,2,128,1,float16,float16,0,0.013704000413417817
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,24,4,128,1,float16,float16,0,0.013876800239086152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,24,8,128,1,float16,float16,0,0.01401119977235794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,24,2,128,1,float16,float16,0,0.014660799503326416
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,24,24,128,1,float16,float16,0,0.014998400211334228
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,24,24,128,1,float16,float16,0,0.014612799882888794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,24,1,128,1,float16,float16,0,0.013944000005722046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,24,4,128,1,float16,float16,0,0.013764800131320953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,24,8,128,1,float16,float16,0,0.013892799615859985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,24,2,128,1,float16,float16,0,0.017473599314689635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,24,4,128,1,float16,float16,0,0.07952319979667663
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,24,2,128,1,float16,float16,0,0.015163199603557586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,24,4,128,1,float16,float16,0,0.014662399888038635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,24,2,128,1,float16,float16,0,0.01372160017490387
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,16,1,128,1,float16,float16,0,20.925917053222655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,16,2,128,1,float16,float16,0,21.292251586914062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,16,4,128,1,float16,float16,0,21.378703308105468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,16,1,128,1,float16,float16,0,10.762687683105469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,16,1,128,1,float16,float16,0,42.170660400390624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,16,2,128,1,float16,float16,0,42.88078918457031
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,16,4,128,1,float16,float16,0,43.27484130859375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,16,8,128,1,float16,float16,0,43.37739562988281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,16,8,128,1,float16,float16,0,21.402064514160156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,16,2,128,1,float16,float16,0,10.783222198486328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,16,2,128,1,float16,float16,0,5.41409912109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,16,1,128,1,float16,float16,0,5.477718353271484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,16,4,128,1,float16,float16,0,10.789539337158203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,16,4,128,1,float16,float16,0,5.6153312683105465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,16,8,128,1,float16,float16,0,11.102699279785156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,16,8,128,1,float16,float16,0,5.641966247558594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,16,1,128,1,float16,float16,0,24.092034912109376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,16,2,128,1,float16,float16,0,24.020600891113283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,16,4,128,1,float16,float16,0,24.267619323730468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,16,8,128,1,float16,float16,0,24.927992248535155
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,16,1,128,1,float16,float16,0,12.153145599365235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,16,2,128,1,float16,float16,0,12.0144287109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,16,4,128,1,float16,float16,0,12.242139434814453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,16,8,128,1,float16,float16,0,12.529039764404297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,16,1,128,1,float16,float16,0,6.074523162841797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,16,2,128,1,float16,float16,0,6.128500747680664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,16,4,128,1,float16,float16,0,6.314900970458984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,16,1,128,1,float16,float16,0,3.156491279602051
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,16,8,128,1,float16,float16,0,6.31943359375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,16,2,128,1,float16,float16,0,3.217784118652344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,16,4,128,1,float16,float16,0,3.260478210449219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,16,8,128,1,float16,float16,0,3.3349681854248048
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,16,1,128,1,float16,float16,0,16.790834045410158
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,16,2,128,1,float16,float16,0,16.963465881347656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,16,4,128,1,float16,float16,0,16.901422119140626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,16,8,128,1,float16,float16,0,17.37316436767578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,16,1,128,1,float16,float16,0,8.51415557861328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,16,2,128,1,float16,float16,0,8.543927764892578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,16,4,128,1,float16,float16,0,8.664656066894532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,16,8,128,1,float16,float16,0,8.800955200195313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,16,16,128,1,float16,float16,0,3.3339664459228517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,16,16,128,1,float16,float16,0,6.456703948974609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,16,16,128,1,float16,float16,0,4.571944046020508
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,16,16,128,1,float16,float16,0,5.866727828979492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,16,1,128,1,float16,float16,0,4.211727905273437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,16,2,128,1,float16,float16,0,4.333219146728515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,16,4,128,1,float16,float16,0,4.391563034057617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,16,1,128,1,float16,float16,0,2.220088005065918
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,16,2,128,1,float16,float16,0,2.2925920486450195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,16,16,128,1,float16,float16,0,2.4210704803466796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,16,16,128,1,float16,float16,0,8.878241729736327
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,16,8,128,1,float16,float16,0,4.4701393127441404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,16,16,128,1,float16,float16,0,12.54125747680664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,16,4,128,1,float16,float16,0,2.3427391052246094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,16,8,128,1,float16,float16,0,2.3662256240844726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,16,16,128,1,float16,float16,0,11.316524505615234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,16,16,128,1,float16,float16,0,22.04585418701172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,16,1,128,1,float16,float16,0,10.887257385253907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,16,16,128,1,float16,float16,0,11.609694671630859
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,16,2,128,1,float16,float16,0,10.811033630371094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,16,4,128,1,float16,float16,0,10.835972595214844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,16,1,128,1,float16,float16,0,21.362350463867188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,16,2,128,1,float16,float16,0,21.458657836914064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,16,1,128,1,float16,float16,0,5.411868667602539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,16,4,128,1,float16,float16,0,21.94770050048828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,16,8,128,1,float16,float16,0,21.757994079589842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,16,2,128,1,float16,float16,0,5.4290321350097654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,16,16,128,1,float16,float16,0,3.0288000106811523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,16,16,128,1,float16,float16,0,5.854830551147461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,16,1,128,1,float16,float16,0,2.7867759704589843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,16,2,128,1,float16,float16,0,2.7814687728881835
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,16,4,128,1,float16,float16,0,5.536001586914063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,16,16,128,1,float16,float16,0,1.6107248306274413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,16,8,128,1,float16,float16,0,11.227094268798828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,16,8,128,1,float16,float16,0,5.6897216796875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,16,8,128,1,float16,float16,0,2.928363227844238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,16,1,128,1,float16,float16,0,1.502894401550293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,16,2,128,1,float16,float16,0,1.4935855865478516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,16,4,128,1,float16,float16,0,1.6360895156860351
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,16,8,128,1,float16,float16,0,1.5684896469116212
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,16,4,128,1,float16,float16,0,2.8514928817749023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,16,1,128,1,float16,float16,0,6.134243011474609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,16,16,128,1,float16,float16,0,6.6620338439941404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,16,2,128,1,float16,float16,0,6.2022560119628904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,16,4,128,1,float16,float16,0,6.1886848449707035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,16,1,128,1,float16,float16,0,3.132428741455078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,16,16,128,1,float16,float16,0,3.374520111083984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,16,1,128,1,float16,float16,0,12.313143920898437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,16,2,128,1,float16,float16,0,12.279110717773438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,16,2,128,1,float16,float16,0,3.1166240692138674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,16,4,128,1,float16,float16,0,12.432337951660156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,16,8,128,1,float16,float16,0,12.936715698242187
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,16,1,128,1,float16,float16,0,1.6288368225097656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,16,16,128,1,float16,float16,0,1.8214303970336914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,16,4,128,1,float16,float16,0,3.200494384765625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,16,2,128,1,float16,float16,0,1.6458368301391602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,16,8,128,1,float16,float16,0,3.1902959823608397
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,16,4,128,1,float16,float16,0,1.6886287689208985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,16,16,128,1,float16,float16,0,1.0030207633972168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,16,2,128,1,float16,float16,0,0.9170144081115723
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,16,4,128,1,float16,float16,0,0.9416720390319824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,16,8,128,1,float16,float16,0,0.9782591819763183
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,16,8,128,1,float16,float16,0,1.7128335952758789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,16,8,128,1,float16,float16,0,6.38280143737793
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,16,1,128,1,float16,float16,0,0.8895919799804688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,16,1,128,1,float16,float16,0,5.6133262634277346
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,16,16,128,1,float16,float16,0,6.192779159545898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,16,2,128,1,float16,float16,0,5.572323226928711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,16,4,128,1,float16,float16,0,5.716033554077148
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,16,16,128,1,float16,float16,0,3.13690071105957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,16,1,128,1,float16,float16,0,11.078202819824218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,16,2,128,1,float16,float16,0,11.14632797241211
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,16,1,128,1,float16,float16,0,2.782222366333008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,16,4,128,1,float16,float16,0,11.385302734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,16,8,128,1,float16,float16,0,11.54918212890625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,16,2,128,1,float16,float16,0,2.864179229736328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,16,8,128,1,float16,float16,0,5.913647842407227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,16,1,128,1,float16,float16,0,1.4582015991210937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,16,16,128,1,float16,float16,0,1.642843246459961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,16,4,128,1,float16,float16,0,2.821227264404297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,16,2,128,1,float16,float16,0,1.4908687591552734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,16,8,128,1,float16,float16,0,2.936374473571777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,16,16,128,1,float16,float16,0,0.9343119621276855
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,16,1,128,1,float16,float16,0,0.7837408065795899
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,16,2,128,1,float16,float16,0,0.807431983947754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,16,4,128,1,float16,float16,0,0.8366991996765136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,16,8,128,1,float16,float16,0,1.5392127990722657
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,16,8,128,1,float16,float16,0,0.8291119575500489
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,16,2,128,1,float16,float16,0,0.4554751873016357
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,16,1,128,1,float16,float16,0,0.4649680137634277
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,16,4,128,1,float16,float16,0,0.46743202209472656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,16,8,128,1,float16,float16,0,0.4907087802886963
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,16,4,128,1,float16,float16,0,1.5107808113098145
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,16,16,128,1,float16,float16,0,0.5416624069213867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,16,16,128,1,float16,float16,0,3.693745422363281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,16,1,128,1,float16,float16,0,3.0750911712646483
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,16,1,128,1,float16,float16,0,6.399343872070313
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,16,2,128,1,float16,float16,0,6.510282897949219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,16,4,128,1,float16,float16,0,6.499288177490234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,16,8,128,1,float16,float16,0,6.865187072753907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,16,4,128,1,float16,float16,0,3.2565486907958983
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,16,1,128,1,float16,float16,0,1.6509344100952148
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,16,2,128,1,float16,float16,0,1.6705360412597656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,16,16,128,1,float16,float16,0,1.906991958618164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,16,4,128,1,float16,float16,0,1.7098560333251953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,16,8,128,1,float16,float16,0,1.745844841003418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,16,1,128,1,float16,float16,0,0.8636848449707031
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,16,16,128,1,float16,float16,0,1.004209613800049
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,16,8,128,1,float16,float16,0,3.3804767608642576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,16,2,128,1,float16,float16,0,0.8779312133789062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,16,4,128,1,float16,float16,0,0.8610112190246582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,16,8,128,1,float16,float16,0,0.9265119552612304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,16,1,128,1,float16,float16,0,0.4727168083190918
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,16,2,128,1,float16,float16,0,0.49182882308959963
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,16,16,128,1,float16,float16,0,0.5702784061431885
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,16,16,128,1,float16,float16,0,0.31836159229278566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,16,8,128,1,float16,float16,0,0.5193535804748535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,16,2,128,1,float16,float16,0,0.30466399192810056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,16,4,128,1,float16,float16,0,0.3004928112030029
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,16,2,128,1,float16,float16,0,3.181939125061035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,16,8,128,1,float16,float16,0,0.31132481098175047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,16,1,128,1,float16,float16,0,2.8818063735961914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,16,4,128,1,float16,float16,0,0.5141503810882568
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,16,1,128,1,float16,float16,0,0.2951024055480957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,16,16,128,1,float16,float16,0,3.602529525756836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,16,1,128,1,float16,float16,0,5.691243362426758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,16,2,128,1,float16,float16,0,5.956044769287109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,16,4,128,1,float16,float16,0,6.027791976928711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,16,2,128,1,float16,float16,0,2.970688056945801
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,16,16,128,1,float16,float16,0,1.831648063659668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,16,4,128,1,float16,float16,0,2.9724752426147463
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,16,8,128,1,float16,float16,0,6.422140502929688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,16,8,128,1,float16,float16,0,3.1946880340576174
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,16,1,128,1,float16,float16,0,1.515236759185791
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,16,16,128,1,float16,float16,0,0.9517359733581543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,16,1,128,1,float16,float16,0,0.8089391708374023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,16,2,128,1,float16,float16,0,1.5129088401794433
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,16,2,128,1,float16,float16,0,0.813212776184082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,16,4,128,1,float16,float16,0,0.8470303535461425
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,16,4,128,1,float16,float16,0,1.5519776344299316
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,16,1,128,1,float16,float16,0,0.41986880302429197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,16,8,128,1,float16,float16,0,1.617193603515625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,16,16,128,1,float16,float16,0,0.5142208099365234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,16,2,128,1,float16,float16,0,0.44213438034057617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,16,8,128,1,float16,float16,0,0.8451824188232422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,16,16,128,1,float16,float16,0,0.31360321044921874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,16,1,128,1,float16,float16,0,0.24522559642791747
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,16,8,128,1,float16,float16,0,0.4753888130187988
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,16,4,128,1,float16,float16,0,0.2607295989990234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,16,2,128,1,float16,float16,0,0.25659520626068116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,16,8,128,1,float16,float16,0,0.2747999906539917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,16,16,128,1,float16,float16,0,0.18294880390167237
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,16,1,128,1,float16,float16,0,0.17305279970169068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,16,2,128,1,float16,float16,0,0.16880639791488647
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,16,4,128,1,float16,float16,0,0.16906239986419677
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,16,8,128,1,float16,float16,0,0.17213599681854247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,16,1,128,1,float16,float16,0,1.677012825012207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,16,2,128,1,float16,float16,0,1.6960927963256835
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,16,16,128,1,float16,float16,0,2.1787439346313477
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,16,4,128,1,float16,float16,0,0.44463357925415037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,16,1,128,1,float16,float16,0,3.3806480407714843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,16,2,128,1,float16,float16,0,3.4700080871582033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,16,4,128,1,float16,float16,0,3.5043582916259766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,16,16,128,1,float16,float16,0,1.1427167892456054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,16,4,128,1,float16,float16,0,1.8061071395874024
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,16,8,128,1,float16,float16,0,3.7848751068115236
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,16,1,128,1,float16,float16,0,0.8508031845092774
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,16,8,128,1,float16,float16,0,1.9165023803710937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,16,2,128,1,float16,float16,0,0.8998527526855469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,16,1,128,1,float16,float16,0,0.4958608150482178
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,16,16,128,1,float16,float16,0,0.5971792221069336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,16,4,128,1,float16,float16,0,0.9543071746826172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,16,2,128,1,float16,float16,0,0.4842368125915527
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,16,4,128,1,float16,float16,0,0.4829840183258057
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,16,1,128,1,float16,float16,0,0.2750175952911377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,16,16,128,1,float16,float16,0,0.3360127925872803
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,16,2,128,1,float16,float16,0,0.26676480770111083
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,16,4,128,1,float16,float16,0,0.28399200439453126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,16,16,128,1,float16,float16,0,0.1987280011177063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,16,8,128,1,float16,float16,0,0.29390559196472166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,16,2,128,1,float16,float16,0,0.1608207941055298
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,16,4,128,1,float16,float16,0,0.1589568018913269
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,16,16,128,1,float16,float16,0,0.13143199682235718
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,16,8,128,1,float16,float16,0,0.17814879417419432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,16,2,128,1,float16,float16,0,0.11029599905014038
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,16,4,128,1,float16,float16,0,0.10485759973526002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,16,1,128,1,float16,float16,0,0.0938543975353241
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,16,8,128,1,float16,float16,0,0.0976032018661499
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,16,8,128,1,float16,float16,0,0.9975215911865234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,16,8,128,1,float16,float16,0,0.5362448215484619
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,16,1,128,1,float16,float16,0,0.16906559467315674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,16,1,128,1,float16,float16,0,3.213286590576172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,16,2,128,1,float16,float16,0,3.254956817626953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,16,16,128,1,float16,float16,0,2.281305694580078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,16,4,128,1,float16,float16,0,3.375593566894531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,16,1,128,1,float16,float16,0,1.546513557434082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,16,2,128,1,float16,float16,0,1.5978303909301759
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,16,8,128,1,float16,float16,0,3.724756622314453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,16,4,128,1,float16,float16,0,1.6731103897094726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,16,16,128,1,float16,float16,0,1.1842191696166993
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,16,8,128,1,float16,float16,0,1.8990863800048827
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,16,1,128,1,float16,float16,0,0.8381952285766602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,16,2,128,1,float16,float16,0,0.8418928146362304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,16,4,128,1,float16,float16,0,0.8695247650146485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,16,16,128,1,float16,float16,0,0.6267199993133545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,16,4,128,1,float16,float16,0,0.46582717895507814
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,16,1,128,1,float16,float16,0,0.4366879940032959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,16,16,128,1,float16,float16,0,0.338153600692749
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,16,8,128,1,float16,float16,0,0.9841343879699707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,16,8,128,1,float16,float16,0,0.4990079879760742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,16,1,128,1,float16,float16,0,0.25162079334259035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,16,2,128,1,float16,float16,0,0.15037920475006103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,16,8,128,1,float16,float16,0,0.27451519966125487
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,16,2,128,1,float16,float16,0,0.2437648057937622
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,16,4,128,1,float16,float16,0,0.2529423952102661
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,16,1,128,1,float16,float16,0,0.136518394947052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,16,16,128,1,float16,float16,0,0.19342880249023436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,16,4,128,1,float16,float16,0,0.14986399412155152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,16,1,128,1,float16,float16,0,0.09880959987640381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,16,8,128,1,float16,float16,0,0.1605936050415039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,16,2,128,1,float16,float16,0,0.09825119972229004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,16,4,128,1,float16,float16,0,0.09567679762840271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,16,8,128,1,float16,float16,0,0.10120639801025391
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,16,1,128,1,float16,float16,0,0.05958880186080932
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,16,2,128,1,float16,float16,0,0.06315360069274903
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,16,4,128,1,float16,float16,0,0.07118560075759887
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,16,8,128,1,float16,float16,0,0.07311360239982605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,16,2,128,1,float16,float16,0,0.4579311847686768
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,16,16,128,1,float16,float16,0,0.11548800468444824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,16,16,128,1,float16,float16,0,0.07379199862480164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,16,1,128,1,float16,float16,0,3.0324111938476563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,16,2,128,1,float16,float16,0,3.0916927337646483
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,16,4,128,1,float16,float16,0,3.2875118255615234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,16,8,128,1,float16,float16,0,3.7683742523193358
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,16,16,128,1,float16,float16,0,2.389614486694336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,16,2,128,1,float16,float16,0,1.5520544052124023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,16,1,128,1,float16,float16,0,1.5163087844848633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,16,4,128,1,float16,float16,0,1.6498655319213866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,16,1,128,1,float16,float16,0,0.7749872207641602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,16,16,128,1,float16,float16,0,1.204075241088867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,16,4,128,1,float16,float16,0,0.8354000091552735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,16,2,128,1,float16,float16,0,0.7917520046234131
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,16,8,128,1,float16,float16,0,1.8908336639404297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,16,1,128,1,float16,float16,0,0.3984064102172852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,16,8,128,1,float16,float16,0,0.9417887687683105
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,16,1,128,1,float16,float16,0,0.21097280979156494
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,16,16,128,1,float16,float16,0,0.3151952028274536
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,16,4,128,1,float16,float16,0,0.43247199058532715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,16,2,128,1,float16,float16,0,0.21533119678497314
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,16,8,128,1,float16,float16,0,0.4862720012664795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,16,16,128,1,float16,float16,0,0.6082784175872803
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,16,4,128,1,float16,float16,0,0.22656800746917724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,16,1,128,1,float16,float16,0,0.11733119487762451
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,16,4,128,1,float16,float16,0,0.12488479614257812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,16,8,128,1,float16,float16,0,0.25495359897613523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,16,2,128,1,float16,float16,0,0.11897759437561035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,16,8,128,1,float16,float16,0,0.13916319608688354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,16,16,128,1,float16,float16,0,0.09610080122947692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,16,1,128,1,float16,float16,0,0.06902719736099243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,16,4,128,1,float16,float16,0,0.07449600100517273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,16,2,128,1,float16,float16,0,0.07009919881820678
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,16,16,128,1,float16,float16,0,0.058531200885772704
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,16,8,128,1,float16,float16,0,0.08054239749908447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,16,1,128,1,float16,float16,0,0.043156799674034116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,16,8,128,1,float16,float16,0,0.045396798849105836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,16,4,128,1,float16,float16,0,0.04387679994106293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,16,1,128,1,float16,float16,0,0.04821600019931793
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,16,2,128,1,float16,float16,0,0.04819360077381134
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,16,4,128,1,float16,float16,0,0.05206720232963562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,16,8,128,1,float16,float16,0,0.05385439991950989
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,16,1,128,1,float16,float16,0,0.6458816051483154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,16,2,128,1,float16,float16,0,0.6870160102844238
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,16,4,128,1,float16,float16,0,0.7671760082244873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,16,8,128,1,float16,float16,0,0.9643487930297852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,16,1,128,1,float16,float16,0,0.33609280586242674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,16,2,128,1,float16,float16,0,0.3569952011108398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,16,16,128,1,float16,float16,0,0.7390399932861328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,16,4,128,1,float16,float16,0,0.4024240016937256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,16,1,128,1,float16,float16,0,0.18086719512939453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,16,16,128,1,float16,float16,0,0.38180639743804934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,16,2,128,1,float16,float16,0,0.4081200122833252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,16,8,128,1,float16,float16,0,0.49081921577453613
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,16,16,128,1,float16,float16,0,0.1690111994743347
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,16,4,128,1,float16,float16,0,0.21467199325561523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,16,8,128,1,float16,float16,0,0.2600928068161011
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,16,1,128,1,float16,float16,0,0.10310879945755005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,16,16,128,1,float16,float16,0,0.21156799793243408
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,16,2,128,1,float16,float16,0,0.10770239830017089
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,16,4,128,1,float16,float16,0,0.11706240177154541
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,16,8,128,1,float16,float16,0,0.14672640562057496
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,16,1,128,1,float16,float16,0,0.064300799369812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,16,2,128,1,float16,float16,0,0.06577919721603394
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,16,16,128,1,float16,float16,0,0.11931519508361817
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,16,2,128,1,float16,float16,0,0.0438511997461319
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,16,16,128,1,float16,float16,0,0.065174400806427
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,16,8,128,1,float16,float16,0,0.08123360276222229
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,16,2,128,1,float16,float16,0,0.03804160058498383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,16,16,128,1,float16,float16,0,0.06049280166625977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,16,1,128,1,float16,float16,0,0.038710400462150574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,16,4,128,1,float16,float16,0,0.041659200191497804
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,16,8,128,1,float16,float16,0,0.049163201451301576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,16,16,128,1,float16,float16,0,0.03772160112857818
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,16,8,128,1,float16,float16,0,0.03248479962348938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,16,16,128,1,float16,float16,0,0.02611680030822754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,16,1,128,1,float16,float16,0,0.022777600586414336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,16,2,128,1,float16,float16,0,0.023401600122451783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,16,4,128,1,float16,float16,0,0.023390400409698486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,16,8,128,1,float16,float16,0,0.024014399945735933
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,16,16,128,1,float16,float16,0,0.024556800723075867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,16,1,128,1,float16,float16,0,0.022375999391078948
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,16,2,128,1,float16,float16,0,0.02263839989900589
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,16,4,128,1,float16,float16,0,0.023049600422382355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,16,8,128,1,float16,float16,0,0.023051199316978455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,16,1,128,1,float16,float16,0,0.26975359916687014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,16,2,128,1,float16,float16,0,0.2944688081741333
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,16,4,128,1,float16,float16,0,0.34582080841064455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,16,8,128,1,float16,float16,0,0.44771518707275393
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,16,16,128,1,float16,float16,0,0.3688015937805176
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,16,1,128,1,float16,float16,0,0.14699039459228516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,16,2,128,1,float16,float16,0,0.16255520582199096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,16,4,128,1,float16,float16,0,0.070169597864151
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,16,4,128,1,float16,float16,0,0.18611359596252441
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,16,8,128,1,float16,float16,0,0.23761439323425293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,16,1,128,1,float16,float16,0,0.0838479995727539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,16,16,128,1,float16,float16,0,0.19859520196914673
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,16,1,128,1,float16,float16,0,0.029422399401664735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,16,4,128,1,float16,float16,0,0.028220799565315247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,16,4,128,1,float16,float16,0,0.10476160049438477
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,16,8,128,1,float16,float16,0,0.13163039684295655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,16,1,128,1,float16,float16,0,0.05257920026779175
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,16,2,128,1,float16,float16,0,0.0557807981967926
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,16,4,128,1,float16,float16,0,0.06152480244636536
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,16,16,128,1,float16,float16,0,0.05888479948043823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,16,1,128,1,float16,float16,0,0.029865598678588866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,16,2,128,1,float16,float16,0,0.03003840148448944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,16,4,128,1,float16,float16,0,0.03154560029506683
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,16,8,128,1,float16,float16,0,0.042684799432754515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,16,16,128,1,float16,float16,0,0.02916640043258667
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,16,1,128,1,float16,float16,0,0.022694399952888487
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,16,2,128,1,float16,float16,0,0.021857599914073943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,16,4,128,1,float16,float16,0,0.02343199998140335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,16,8,128,1,float16,float16,0,0.024326400458812715
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,16,16,128,1,float16,float16,0,0.020718400180339814
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,16,1,128,1,float16,float16,0,0.018140800297260284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,16,2,128,1,float16,float16,0,0.018296000361442567
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,16,4,128,1,float16,float16,0,0.018433600664138794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,16,8,128,1,float16,float16,0,0.018963199853897095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,16,16,128,1,float16,float16,0,0.019310399889945984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,16,1,128,1,float16,float16,0,0.01770080029964447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,16,2,128,1,float16,float16,0,0.01775040030479431
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,16,4,128,1,float16,float16,0,0.017982399463653563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,16,8,128,1,float16,float16,0,0.018011200428009033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,16,16,128,1,float16,float16,0,0.018681600689888
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,16,1,128,1,float16,float16,0,0.017472000420093538
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,16,2,128,1,float16,float16,0,0.017643199861049653
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,16,4,128,1,float16,float16,0,0.01748960018157959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,16,8,128,1,float16,float16,0,0.017632000148296356
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,16,2,128,1,float16,float16,0,0.1918895959854126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,16,1,128,1,float16,float16,0,0.13863999843597413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,16,2,128,1,float16,float16,0,0.15383520126342773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,16,4,128,1,float16,float16,0,0.1811087965965271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,16,8,128,1,float16,float16,0,0.23403520584106446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,16,2,128,1,float16,float16,0,0.028984001278877257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,16,2,128,1,float16,float16,0,0.09173439741134644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,16,16,128,1,float16,float16,0,0.11075839996337891
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,16,8,128,1,float16,float16,0,0.07366399765014649
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,16,1,128,1,float16,float16,0,0.08227519989013672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,16,16,128,1,float16,float16,0,0.1949679970741272
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,16,16,128,1,float16,float16,0,0.10824799537658691
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,16,2,128,1,float16,float16,0,0.08888800144195556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,16,1,128,1,float16,float16,0,0.0499343991279602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,16,8,128,1,float16,float16,0,0.12973439693450928
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,16,4,128,1,float16,float16,0,0.05940319895744324
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,16,2,128,1,float16,float16,0,0.05295360088348389
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,16,8,128,1,float16,float16,0,0.0726639986038208
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,16,2,128,1,float16,float16,0,0.028174400329589844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,16,1,128,1,float16,float16,0,0.02815040051937103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,16,4,128,1,float16,float16,0,0.02966879904270172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,16,16,128,1,float16,float16,0,0.02934719920158386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,16,8,128,1,float16,float16,0,0.04095520079135895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,16,2,128,1,float16,float16,0,0.018873600661754607
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,16,4,128,1,float16,float16,0,0.01937119960784912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,16,1,128,1,float16,float16,0,0.01870719939470291
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,16,8,128,1,float16,float16,0,0.020524799823760986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,16,1,128,1,float16,float16,0,0.01589120030403137
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,16,16,128,1,float16,float16,0,0.019148799777030944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,16,4,128,1,float16,float16,0,0.01608159989118576
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,16,8,128,1,float16,float16,0,0.016332800686359405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,16,2,128,1,float16,float16,0,0.015892800688743592
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,16,16,128,1,float16,float16,0,0.016182400286197662
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,16,1,128,1,float16,float16,0,0.015328000485897064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,16,2,128,1,float16,float16,0,0.015131199359893798
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,16,16,128,1,float16,float16,0,0.015358400344848634
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,16,8,128,1,float16,float16,0,0.015531200170516967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,16,1,128,1,float16,float16,0,0.01512800008058548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,16,2,128,1,float16,float16,0,0.01523679941892624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,16,4,128,1,float16,float16,0,0.015115199983119965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,16,8,128,1,float16,float16,0,0.015147200226783753
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,16,1,128,1,float16,float16,0,0.014892800152301789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,16,2,128,1,float16,float16,0,0.014971199631690978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,16,4,128,1,float16,float16,0,0.014825600385665893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,16,8,128,1,float16,float16,0,0.015115199983119965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,16,1,128,1,float16,float16,0,0.08199359774589539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,16,2,128,1,float16,float16,0,0.08903520107269287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,16,4,128,1,float16,float16,0,0.10124319791793823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,16,8,128,1,float16,float16,0,0.14626400470733641
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,16,1,128,1,float16,float16,0,0.05173280239105225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,16,16,128,1,float16,float16,0,0.13238400220870972
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,16,2,128,1,float16,float16,0,0.054230397939682005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,16,8,128,1,float16,float16,0,0.08144320249557495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,16,16,128,1,float16,float16,0,0.07217599749565125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,16,1,128,1,float16,float16,0,0.027723199129104613
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,16,2,128,1,float16,float16,0,0.02844479978084564
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,16,4,128,1,float16,float16,0,0.030215999484062193
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,16,8,128,1,float16,float16,0,0.047023999691009524
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,16,1,128,1,float16,float16,0,0.019046400487422944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,16,2,128,1,float16,float16,0,0.019281600415706635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,16,4,128,1,float16,float16,0,0.019734400510787963
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,16,8,128,1,float16,float16,0,0.02624480128288269
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,16,16,128,1,float16,float16,0,0.024979199469089507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,16,4,128,1,float16,float16,0,0.10334559679031372
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,16,1,128,1,float16,float16,0,0.015659199655056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,16,16,128,1,float16,float16,0,0.059139198064804076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,16,4,128,1,float16,float16,0,0.016017599403858183
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,16,2,128,1,float16,float16,0,0.015803200006484986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,16,8,128,1,float16,float16,0,0.018214400112628936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,16,16,128,1,float16,float16,0,0.017987200617790224
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,16,1,128,1,float16,float16,0,0.015192000567913056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,16,2,128,1,float16,float16,0,0.01520799994468689
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,16,4,128,1,float16,float16,0,0.015430399775505066
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,16,8,128,1,float16,float16,0,0.015535999834537507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,16,16,128,1,float16,float16,0,0.01539199948310852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,16,1,128,1,float16,float16,0,0.014815999567508698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,16,2,128,1,float16,float16,0,0.015091200172901154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,16,4,128,1,float16,float16,0,0.015067200362682342
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,16,8,128,1,float16,float16,0,0.014828799664974213
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,16,16,128,1,float16,float16,0,0.014945599436759948
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,16,1,128,1,float16,float16,0,0.014824000000953675
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,16,2,128,1,float16,float16,0,0.01478559970855713
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,16,4,128,1,float16,float16,0,0.014801600575447082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,16,8,128,1,float16,float16,0,0.014563199877738953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,16,16,128,1,float16,float16,0,0.014735999703407287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,16,4,128,1,float16,float16,0,0.01547040045261383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,16,2,128,1,float16,float16,0,0.014912000298500061
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,16,4,128,1,float16,float16,0,0.014636799693107605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,16,8,128,1,float16,float16,0,0.014448000490665436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,16,16,128,1,float16,float16,0,0.015216000378131866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,16,1,128,1,float16,float16,0,0.052907198667526245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,16,4,128,1,float16,float16,0,0.07206879854202271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,16,16,128,1,float16,float16,0,0.10950080156326295
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,16,8,128,1,float16,float16,0,0.11562399864196778
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,16,2,128,1,float16,float16,0,0.029576000571250916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,16,8,128,1,float16,float16,0,0.06262400150299072
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,16,4,128,1,float16,float16,0,0.05969600081443786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,16,16,128,1,float16,float16,0,0.05467519760131836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,16,1,128,1,float16,float16,0,0.01974560022354126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,16,2,128,1,float16,float16,0,0.01974080055952072
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,16,4,128,1,float16,float16,0,0.025772801041603087
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,16,8,128,1,float16,float16,0,0.033825600147247316
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,16,16,128,1,float16,float16,0,0.03312320113182068
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,16,16,128,1,float16,float16,0,0.03779360055923462
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,16,1,128,1,float16,float16,0,0.016041600704193117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,16,2,128,1,float16,float16,0,0.016366399824619293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,16,4,128,1,float16,float16,0,0.018212799727916718
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,16,8,128,1,float16,float16,0,0.024414399266242982
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,16,16,128,1,float16,float16,0,0.023664000630378722
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,16,1,128,1,float16,float16,0,0.015113599598407745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,16,2,128,1,float16,float16,0,0.015190400183200836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,16,4,128,1,float16,float16,0,0.015174399316310882
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,16,8,128,1,float16,float16,0,0.017291200160980225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,16,16,128,1,float16,float16,0,0.01717119961977005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,16,1,128,1,float16,float16,0,0.014988799393177033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,16,2,128,1,float16,float16,0,0.015007999539375306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,16,4,128,1,float16,float16,0,0.01478240042924881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,16,8,128,1,float16,float16,0,0.015078400075435639
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,16,16,128,1,float16,float16,0,0.015006400644779205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,16,1,128,1,float16,float16,0,0.014767999947071075
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,16,2,128,1,float16,float16,0,0.014732800424098969
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,16,4,128,1,float16,float16,0,0.01446560025215149
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,16,16,128,1,float16,float16,0,0.01478240042924881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,16,1,128,1,float16,float16,0,0.014657600224018097
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,16,2,128,1,float16,float16,0,0.014800000190734863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,16,4,128,1,float16,float16,0,0.014201599359512328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,16,16,128,1,float16,float16,0,0.014519999921321868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,16,1,128,1,float16,float16,0,0.014628799259662628
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,16,2,128,1,float16,float16,0,0.01464959979057312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,16,4,128,1,float16,float16,0,0.014379200339317322
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,16,8,128,1,float16,float16,0,0.014116799831390381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,16,1,128,1,float16,float16,0,0.02624799907207489
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,16,1,128,1,float16,float16,0,0.014707200229167938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,16,2,128,1,float16,float16,0,0.0339711993932724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,16,4,128,1,float16,float16,0,0.05162559747695923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,16,2,128,1,float16,float16,0,0.05514879822731018
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,16,8,128,1,float16,float16,0,0.08921759724617004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,16,16,128,1,float16,float16,0,0.08659520149230956
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,16,2,128,1,float16,float16,0,0.024609600007534028
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,16,1,128,1,float16,float16,0,0.029246398806571962
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,16,4,128,1,float16,float16,0,0.03789919912815094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,16,4,128,1,float16,float16,0,0.03214559853076935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,16,8,128,1,float16,float16,0,0.04988479912281037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,16,16,128,1,float16,float16,0,0.04854240119457245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,16,2,128,1,float16,float16,0,0.01740639954805374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,16,1,128,1,float16,float16,0,0.01595360040664673
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,16,4,128,1,float16,float16,0,0.023416000604629516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,16,16,128,1,float16,float16,0,0.03060159981250763
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,16,1,128,1,float16,float16,0,0.014963200688362122
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,16,2,128,1,float16,float16,0,0.015028800070285796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,16,4,128,1,float16,float16,0,0.016927999258041383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,16,8,128,1,float16,float16,0,0.02300959974527359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,16,16,128,1,float16,float16,0,0.022313599288463593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,16,1,128,1,float16,float16,0,0.0141744002699852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,16,2,128,1,float16,float16,0,0.014457599818706512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,16,4,128,1,float16,float16,0,0.014472000300884247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,16,8,128,1,float16,float16,0,0.016228799521923066
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,16,16,128,1,float16,float16,0,0.01666239947080612
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,16,1,128,1,float16,float16,0,0.01387840062379837
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,16,2,128,1,float16,float16,0,0.0141744002699852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,16,4,128,1,float16,float16,0,0.01443839967250824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,16,8,128,1,float16,float16,0,0.014316800236701965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,16,16,128,1,float16,float16,0,0.014531199634075165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,16,1,128,1,float16,float16,0,0.014207999408245086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,16,2,128,1,float16,float16,0,0.013843199610710144
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,16,4,128,1,float16,float16,0,0.01387840062379837
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,16,8,128,1,float16,float16,0,0.01446239948272705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,16,16,128,1,float16,float16,0,0.014347200095653535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,16,1,128,1,float16,float16,0,0.013779200613498688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,16,2,128,1,float16,float16,0,0.014217600226402283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,16,4,128,1,float16,float16,0,0.013838399946689606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,16,8,128,1,float16,float16,0,0.014102399349212646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,16,16,128,1,float16,float16,0,0.014696000516414643
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,16,1,128,1,float16,float16,0,0.013775999844074249
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,16,2,128,1,float16,float16,0,0.01374559998512268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,16,4,128,1,float16,float16,0,0.01419519931077957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,16,8,128,1,float16,float16,0,0.014035199582576752
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,16,8,128,1,float16,float16,0,0.014550399780273438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,16,8,128,1,float16,float16,0,0.014555199444293976
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,16,1,128,1,float16,float16,0,0.018479999899864197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,16,8,128,1,float16,float16,0,0.030934399366378783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,12,1,128,1,float16,float16,0,15.893060302734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,12,2,128,1,float16,float16,0,16.107078552246094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,12,4,128,1,float16,float16,0,16.31835174560547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,12,1,128,1,float16,float16,0,8.076387023925781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,12,2,128,1,float16,float16,0,8.121451568603515
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,12,4,128,1,float16,float16,0,8.382599639892579
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,12,1,128,1,float16,float16,0,4.143641662597656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,12,2,128,1,float16,float16,0,31.304098510742186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,12,1,128,1,float16,float16,0,32.05361633300781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,12,4,128,1,float16,float16,0,32.14653625488281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,12,4,128,1,float16,float16,0,4.363556671142578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,12,2,128,1,float16,float16,0,4.273759841918945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,12,1,128,1,float16,float16,0,9.110886383056641
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,12,1,128,1,float16,float16,0,17.79058837890625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,12,2,128,1,float16,float16,0,18.49827880859375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,12,4,128,1,float16,float16,0,18.37157745361328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,12,2,128,1,float16,float16,0,9.229841613769532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,12,1,128,1,float16,float16,0,4.622646331787109
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,12,4,128,1,float16,float16,0,9.166686248779296
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,12,2,128,1,float16,float16,0,4.69531021118164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,12,4,128,1,float16,float16,0,4.7913776397705075
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,12,1,128,1,float16,float16,0,2.4230464935302733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,12,2,128,1,float16,float16,0,2.481876754760742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,12,4,128,1,float16,float16,0,2.56734561920166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,12,1,128,1,float16,float16,0,12.570708465576171
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,12,2,128,1,float16,float16,0,12.677410888671876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,12,4,128,1,float16,float16,0,12.817947387695312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,12,1,128,1,float16,float16,0,6.362417602539063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,12,2,128,1,float16,float16,0,6.459572601318359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,12,4,128,1,float16,float16,0,6.557263946533203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,12,12,128,1,float16,float16,0,2.547819137573242
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,12,12,128,1,float16,float16,0,4.99109115600586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,12,12,128,1,float16,float16,0,3.4486465454101562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,12,12,128,1,float16,float16,0,4.431619262695312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,12,1,128,1,float16,float16,0,3.1525360107421876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,12,12,128,1,float16,float16,0,1.9156288146972655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,12,1,128,1,float16,float16,0,1.7255712509155274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,12,2,128,1,float16,float16,0,3.318075180053711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,12,4,128,1,float16,float16,0,3.3643489837646485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,12,12,128,1,float16,float16,0,6.701611328125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,12,2,128,1,float16,float16,0,1.7733951568603517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,12,4,128,1,float16,float16,0,1.8145360946655273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,12,12,128,1,float16,float16,0,8.461231994628907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,12,12,128,1,float16,float16,0,9.630728149414063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,12,12,128,1,float16,float16,0,16.908671569824218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,12,12,128,1,float16,float16,0,8.742572784423828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,12,1,128,1,float16,float16,0,7.995053100585937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,12,2,128,1,float16,float16,0,8.108744049072266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,12,12,128,1,float16,float16,0,4.49743537902832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,12,4,128,1,float16,float16,0,8.3109375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,12,1,128,1,float16,float16,0,4.0981391906738285
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,12,2,128,1,float16,float16,0,4.118899154663086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,12,12,128,1,float16,float16,0,2.3590335845947266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,12,4,128,1,float16,float16,0,4.2164161682128904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,12,1,128,1,float16,float16,0,2.1439983367919924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,12,1,128,1,float16,float16,0,1.155183982849121
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,12,12,128,1,float16,float16,0,1.309216022491455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,12,1,128,1,float16,float16,0,16.270370483398438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,12,2,128,1,float16,float16,0,16.205386352539062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,12,4,128,1,float16,float16,0,16.462757873535157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,12,2,128,1,float16,float16,0,2.221028709411621
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,12,4,128,1,float16,float16,0,2.2107967376708983
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,12,2,128,1,float16,float16,0,1.1798944473266602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,12,4,128,1,float16,float16,0,1.265662384033203
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,12,1,128,1,float16,float16,0,4.53250732421875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,12,12,128,1,float16,float16,0,5.051145553588867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,12,2,128,1,float16,float16,0,4.669166564941406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,12,4,128,1,float16,float16,0,4.758855819702148
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,12,12,128,1,float16,float16,0,2.5561376571655274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,12,1,128,1,float16,float16,0,2.3693872451782227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,12,2,128,1,float16,float16,0,2.429075241088867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,12,12,128,1,float16,float16,0,1.4108624458312988
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,12,1,128,1,float16,float16,0,9.33343048095703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,12,2,128,1,float16,float16,0,9.217566680908202
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,12,1,128,1,float16,float16,0,1.2548928260803223
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,12,2,128,1,float16,float16,0,1.3006624221801757
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,12,4,128,1,float16,float16,0,9.534454345703125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,12,4,128,1,float16,float16,0,1.338155174255371
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,12,12,128,1,float16,float16,0,0.8035087585449219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,12,1,128,1,float16,float16,0,0.7656000137329102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,12,2,128,1,float16,float16,0,0.7488192081451416
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,12,4,128,1,float16,float16,0,0.7715248107910156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,12,4,128,1,float16,float16,0,2.3762880325317384
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,12,1,128,1,float16,float16,0,4.216451263427734
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,12,2,128,1,float16,float16,0,4.192979049682617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,12,4,128,1,float16,float16,0,4.241561508178711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,12,12,128,1,float16,float16,0,4.768355178833008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,12,12,128,1,float16,float16,0,2.33331356048584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,12,12,128,1,float16,float16,0,1.2769871711730958
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,12,1,128,1,float16,float16,0,2.1677488327026366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,12,2,128,1,float16,float16,0,2.150030326843262
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,12,1,128,1,float16,float16,0,1.1292943954467773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,12,4,128,1,float16,float16,0,2.2195648193359374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,12,1,128,1,float16,float16,0,0.6152927875518799
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,12,1,128,1,float16,float16,0,8.267063903808594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,12,2,128,1,float16,float16,0,1.1337311744689942
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,12,2,128,1,float16,float16,0,0.6602096080780029
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,12,4,128,1,float16,float16,0,1.1882032394409179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,12,2,128,1,float16,float16,0,8.434868621826173
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,12,1,128,1,float16,float16,0,0.38374879360198977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,12,12,128,1,float16,float16,0,0.451262378692627
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,12,4,128,1,float16,float16,0,8.620521545410156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,12,4,128,1,float16,float16,0,0.6589248180389404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,12,4,128,1,float16,float16,0,0.3920464038848877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,12,1,128,1,float16,float16,0,2.3540016174316407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,12,12,128,1,float16,float16,0,0.7124911785125733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,12,2,128,1,float16,float16,0,2.4203664779663088
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,12,12,128,1,float16,float16,0,2.7932527542114256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,12,2,128,1,float16,float16,0,0.4051072120666504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,12,12,128,1,float16,float16,0,1.4069631576538086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,12,1,128,1,float16,float16,0,1.279587173461914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,12,2,128,1,float16,float16,0,1.2838000297546386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,12,1,128,1,float16,float16,0,4.798979187011719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,12,4,128,1,float16,float16,0,1.2669919967651366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,12,2,128,1,float16,float16,0,4.854616165161133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,12,4,128,1,float16,float16,0,5.017374420166016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,12,4,128,1,float16,float16,0,2.4651216506958007
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,12,12,128,1,float16,float16,0,0.7623216152191162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,12,1,128,1,float16,float16,0,0.6930992126464843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,12,12,128,1,float16,float16,0,0.4340559959411621
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,12,1,128,1,float16,float16,0,0.402023983001709
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,12,4,128,1,float16,float16,0,0.7316832065582275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,12,2,128,1,float16,float16,0,0.42134718894958495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,12,12,128,1,float16,float16,0,0.26218400001525877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,12,4,128,1,float16,float16,0,0.423415994644165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,12,2,128,1,float16,float16,0,0.25464959144592286
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,12,4,128,1,float16,float16,0,0.2650559902191162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,12,1,128,1,float16,float16,0,2.1340816497802733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,12,2,128,1,float16,float16,0,2.215123176574707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,12,12,128,1,float16,float16,0,2.666916847229004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,12,2,128,1,float16,float16,0,0.685643196105957
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,12,1,128,1,float16,float16,0,0.26163198947906496
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,12,1,128,1,float16,float16,0,1.1226367950439453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,12,12,128,1,float16,float16,0,1.4054256439208985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,12,2,128,1,float16,float16,0,1.1183695793151855
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,12,2,128,1,float16,float16,0,4.4453071594238285
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,12,4,128,1,float16,float16,0,4.625825500488281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,12,4,128,1,float16,float16,0,2.3387680053710938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,12,4,128,1,float16,float16,0,1.1830975532531738
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,12,12,128,1,float16,float16,0,0.7238895893096924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,12,1,128,1,float16,float16,0,0.6183584213256836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,12,2,128,1,float16,float16,0,0.3424704074859619
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,12,12,128,1,float16,float16,0,0.2413696050643921
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,12,1,128,1,float16,float16,0,0.35602080821990967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,12,2,128,1,float16,float16,0,0.60141921043396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,12,12,128,1,float16,float16,0,0.42690238952636717
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,12,4,128,1,float16,float16,0,0.35734241008758544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,12,4,128,1,float16,float16,0,0.6560224056243896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,12,1,128,1,float16,float16,0,0.20865919589996337
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,12,2,128,1,float16,float16,0,0.20504798889160156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,12,12,128,1,float16,float16,0,0.14263360500335692
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,12,4,128,1,float16,float16,0,0.22040319442749023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,12,2,128,1,float16,float16,0,0.1401360034942627
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,12,1,128,1,float16,float16,0,0.1241104006767273
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,12,4,128,1,float16,float16,0,0.13896000385284424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,12,1,128,1,float16,float16,0,1.2610719680786133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,12,2,128,1,float16,float16,0,1.3000703811645509
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,12,12,128,1,float16,float16,0,1.69128475189209
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,12,1,128,1,float16,float16,0,4.413288116455078
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,12,1,128,1,float16,float16,0,2.5040159225463867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,12,2,128,1,float16,float16,0,2.591476821899414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,12,1,128,1,float16,float16,0,0.6791984081268311
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,12,4,128,1,float16,float16,0,2.67968807220459
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,12,12,128,1,float16,float16,0,0.8785216331481933
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,12,2,128,1,float16,float16,0,0.6821119785308838
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,12,4,128,1,float16,float16,0,1.3734416007995605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,12,12,128,1,float16,float16,0,0.2611407995223999
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,12,12,128,1,float16,float16,0,0.4916800022125244
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,12,4,128,1,float16,float16,0,0.7584671974182129
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,12,2,128,1,float16,float16,0,0.3896496057510376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,12,4,128,1,float16,float16,0,0.39668800830841067
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,12,2,128,1,float16,float16,0,0.21918718814849852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,12,12,128,1,float16,float16,0,0.1714400053024292
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,12,2,128,1,float16,float16,0,0.13432320356369018
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,12,1,128,1,float16,float16,0,0.1444416046142578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,12,4,128,1,float16,float16,0,0.13544479608535767
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,12,12,128,1,float16,float16,0,0.12548320293426513
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,12,1,128,1,float16,float16,0,0.08446400165557862
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,12,2,128,1,float16,float16,0,0.09563840031623841
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,12,4,128,1,float16,float16,0,0.11321760416030884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,12,12,128,1,float16,float16,0,1.7221296310424805
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,12,1,128,1,float16,float16,0,2.372871971130371
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,12,2,128,1,float16,float16,0,2.4825664520263673
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,12,1,128,1,float16,float16,0,0.3639647960662842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,12,4,128,1,float16,float16,0,2.7000303268432617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,12,1,128,1,float16,float16,0,0.2175071954727173
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,12,4,128,1,float16,float16,0,0.23504319190979003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,12,1,128,1,float16,float16,0,1.2312560081481934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,12,2,128,1,float16,float16,0,0.6349952220916748
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,12,1,128,1,float16,float16,0,0.6590735912322998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,12,12,128,1,float16,float16,0,0.8812975883483887
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,12,2,128,1,float16,float16,0,1.3062095642089844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,12,12,128,1,float16,float16,0,0.47341442108154297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,12,4,128,1,float16,float16,0,1.386731243133545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,12,1,128,1,float16,float16,0,0.3325488090515137
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,12,4,128,1,float16,float16,0,0.6927216053009033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,12,2,128,1,float16,float16,0,0.36619839668273924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,12,12,128,1,float16,float16,0,0.2738879919052124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,12,1,128,1,float16,float16,0,0.19716479778289794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,12,2,128,1,float16,float16,0,0.20689280033111573
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,12,4,128,1,float16,float16,0,0.20736639499664306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,12,12,128,1,float16,float16,0,0.15481280088424682
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,12,4,128,1,float16,float16,0,0.13152639865875243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,12,12,128,1,float16,float16,0,0.08701599836349487
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,12,1,128,1,float16,float16,0,0.07725440263748169
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,12,4,128,1,float16,float16,0,0.07071200013160706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,12,2,128,1,float16,float16,0,0.07463999986648559
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,12,1,128,1,float16,float16,0,0.05247200131416321
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,12,2,128,1,float16,float16,0,0.053467202186584475
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,12,4,128,1,float16,float16,0,0.05411360263824463
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,12,4,128,1,float16,float16,0,0.3729504108428955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,12,1,128,1,float16,float16,0,0.12337280511856079
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,12,2,128,1,float16,float16,0,0.1258847951889038
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,12,1,128,1,float16,float16,0,2.2986576080322267
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,12,2,128,1,float16,float16,0,2.430001640319824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,12,4,128,1,float16,float16,0,2.6615503311157225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,12,12,128,1,float16,float16,0,0.05809119939804077
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,12,12,128,1,float16,float16,0,1.789616012573242
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,12,1,128,1,float16,float16,0,0.5933008193969727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,12,2,128,1,float16,float16,0,0.6261312007904053
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,12,1,128,1,float16,float16,0,1.1602895736694336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,12,2,128,1,float16,float16,0,1.2271039962768555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,12,12,128,1,float16,float16,0,0.46440801620483396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,12,4,128,1,float16,float16,0,0.679259204864502
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,12,1,128,1,float16,float16,0,0.3100399971008301
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,12,12,128,1,float16,float16,0,0.9080863952636719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,12,2,128,1,float16,float16,0,0.32697598934173583
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,12,1,128,1,float16,float16,0,0.16918400526046753
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,12,4,128,1,float16,float16,0,0.35487680435180663
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,12,12,128,1,float16,float16,0,0.24423038959503174
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,12,2,128,1,float16,float16,0,0.17535680532455444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,12,12,128,1,float16,float16,0,0.13446240425109862
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,12,1,128,1,float16,float16,0,0.09698719978332519
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,12,4,128,1,float16,float16,0,0.19080959558486937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,12,4,128,1,float16,float16,0,1.343073558807373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,12,12,128,1,float16,float16,0,0.07988479733467102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,12,2,128,1,float16,float16,0,0.10047199726104736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,12,4,128,1,float16,float16,0,0.10901600122451782
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,12,1,128,1,float16,float16,0,0.058481597900390626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,12,12,128,1,float16,float16,0,0.04601599872112274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,12,4,128,1,float16,float16,0,0.06514400243759155
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,12,12,128,1,float16,float16,0,0.044095999002456664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,12,4,128,1,float16,float16,0,0.043227198719978335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,12,2,128,1,float16,float16,0,0.041652798652648926
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,12,2,128,1,float16,float16,0,0.041247999668121337
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,12,1,128,1,float16,float16,0,0.04195519983768463
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,12,4,128,1,float16,float16,0,0.04229120016098022
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,12,1,128,1,float16,float16,0,0.5023200035095214
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,12,2,128,1,float16,float16,0,0.552345609664917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,12,1,128,1,float16,float16,0,0.26463520526885986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,12,2,128,1,float16,float16,0,0.28647680282592775
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,12,12,128,1,float16,float16,0,0.5607855796813965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,12,12,128,1,float16,float16,0,0.29781599044799806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,12,4,128,1,float16,float16,0,0.34565439224243166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,12,4,128,1,float16,float16,0,0.6632912158966064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,12,1,128,1,float16,float16,0,0.14447519779205323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,12,4,128,1,float16,float16,0,0.18739360570907593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,12,1,128,1,float16,float16,0,0.08536480069160461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,12,2,128,1,float16,float16,0,0.08972960114479064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,12,4,128,1,float16,float16,0,0.10362880229949951
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,12,1,128,1,float16,float16,0,0.05336959958076477
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,12,12,128,1,float16,float16,0,0.09262400269508361
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,12,2,128,1,float16,float16,0,0.05594720244407654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,12,4,128,1,float16,float16,0,0.06243039965629578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,12,12,128,1,float16,float16,0,0.048891198635101316
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,12,1,128,1,float16,float16,0,0.028433600068092348
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,12,2,128,1,float16,float16,0,0.03208959996700287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,12,4,128,1,float16,float16,0,0.0336432009935379
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,12,12,128,1,float16,float16,0,0.027518400549888612
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,12,1,128,1,float16,float16,0,0.023452800512313843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,12,2,128,1,float16,float16,0,0.023668800294399262
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,12,4,128,1,float16,float16,0,0.024822400510311128
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,12,12,128,1,float16,float16,0,0.025172799825668335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,12,1,128,1,float16,float16,0,0.022841599583625794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,12,2,128,1,float16,float16,0,0.022675199806690215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,12,4,128,1,float16,float16,0,0.023628799617290495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,12,12,128,1,float16,float16,0,0.02435680031776428
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,12,2,128,1,float16,float16,0,0.02205760031938553
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,12,4,128,1,float16,float16,0,0.023425599932670592
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,12,2,128,1,float16,float16,0,0.06280800104141235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,12,1,128,1,float16,float16,0,0.214736008644104
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,12,1,128,1,float16,float16,0,0.041091200709342954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,12,4,128,1,float16,float16,0,0.30297439098358153
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,12,1,128,1,float16,float16,0,0.11781439781188965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,12,12,128,1,float16,float16,0,0.2847007989883423
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,12,2,128,1,float16,float16,0,0.13764959573745728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,12,4,128,1,float16,float16,0,0.16477919816970826
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,12,1,128,1,float16,float16,0,0.07071359753608704
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,12,12,128,1,float16,float16,0,0.15589280128479005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,12,2,128,1,float16,float16,0,0.0771727979183197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,12,4,128,1,float16,float16,0,0.09331039786338806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,12,12,128,1,float16,float16,0,0.08304640054702758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,12,2,128,1,float16,float16,0,0.04485920071601868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,12,1,128,1,float16,float16,0,0.04074879884719849
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,12,4,128,1,float16,float16,0,0.05239840149879456
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,12,12,128,1,float16,float16,0,0.041499200463294986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,12,1,128,1,float16,float16,0,0.02367199957370758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,12,2,128,1,float16,float16,0,0.02398719936609268
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,12,2,128,1,float16,float16,0,0.15548640489578247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,12,4,128,1,float16,float16,0,0.02584800124168396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,12,1,128,1,float16,float16,0,0.018796800076961516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,12,12,128,1,float16,float16,0,0.022603200376033784
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,12,12,128,1,float16,float16,0,0.16728320121765136
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,12,4,128,1,float16,float16,0,0.019713599979877473
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,12,2,128,1,float16,float16,0,0.019099199771881105
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,12,1,128,1,float16,float16,0,0.017998400330543517
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,12,2,128,1,float16,float16,0,0.018003199994564057
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,12,4,128,1,float16,float16,0,0.01868479996919632
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,12,12,128,1,float16,float16,0,0.01945279985666275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,12,1,128,1,float16,float16,0,0.017657600343227386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,12,4,128,1,float16,float16,0,0.018223999440670012
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,12,12,128,1,float16,float16,0,0.018811200559139252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,12,1,128,1,float16,float16,0,0.017411200702190398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,12,2,128,1,float16,float16,0,0.017374399304389953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,12,4,128,1,float16,float16,0,0.018028800189495087
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,12,1,128,1,float16,float16,0,0.11445120573043824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,12,2,128,1,float16,float16,0,0.12919520139694213
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,12,4,128,1,float16,float16,0,0.1637984037399292
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,12,12,128,1,float16,float16,0,0.15343999862670898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,12,2,128,1,float16,float16,0,0.0756335973739624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,12,4,128,1,float16,float16,0,0.09393759965896606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,12,1,128,1,float16,float16,0,0.02245440036058426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,12,12,128,1,float16,float16,0,0.08462880253791809
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,12,1,128,1,float16,float16,0,0.037028801441192624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,12,2,128,1,float16,float16,0,0.042008000612258914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,12,4,128,1,float16,float16,0,0.05467680096626282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,12,12,128,1,float16,float16,0,0.044353601336479184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,12,1,128,1,float16,float16,0,0.020598399639129638
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,12,2,128,1,float16,float16,0,0.021254399418830873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,12,2,128,1,float16,float16,0,0.24044959545135497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,12,4,128,1,float16,float16,0,0.02816160023212433
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,12,12,128,1,float16,float16,0,0.021033599972724915
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,12,1,128,1,float16,float16,0,0.016760000586509706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,12,4,128,1,float16,float16,0,0.019259199500083923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,12,12,128,1,float16,float16,0,0.016926400363445282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,12,2,128,1,float16,float16,0,0.016843199729919434
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,12,2,128,1,float16,float16,0,0.015569600462913512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,12,4,128,1,float16,float16,0,0.015807999670505522
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,12,12,128,1,float16,float16,0,0.015887999534606935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,12,1,128,1,float16,float16,0,0.015059199929237366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,12,2,128,1,float16,float16,0,0.015116800367832185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,12,12,128,1,float16,float16,0,0.020110400021076204
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,12,2,128,1,float16,float16,0,0.017822399735450745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,12,1,128,1,float16,float16,0,0.014791999757289887
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,12,12,128,1,float16,float16,0,0.015263999998569488
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,12,4,128,1,float16,float16,0,0.015278400480747223
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,12,4,128,1,float16,float16,0,0.015390400588512421
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,12,1,128,1,float16,float16,0,0.06763359904289246
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,12,2,128,1,float16,float16,0,0.014776000380516052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,12,2,128,1,float16,float16,0,0.015035200119018554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,12,12,128,1,float16,float16,0,0.01509920060634613
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,12,1,128,1,float16,float16,0,0.014750400185585022
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,12,1,128,1,float16,float16,0,0.06856160163879395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,12,2,128,1,float16,float16,0,0.08013439774513245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,12,4,128,1,float16,float16,0,0.09218720197677613
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,12,1,128,1,float16,float16,0,0.0375791996717453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,12,12,128,1,float16,float16,0,0.10378079414367676
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,12,2,128,1,float16,float16,0,0.0483487993478775
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,12,1,128,1,float16,float16,0,0.021220800280570985
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,12,12,128,1,float16,float16,0,0.05877599716186523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,12,4,128,1,float16,float16,0,0.028300800919532777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,12,2,128,1,float16,float16,0,0.027209600806236266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,12,12,128,1,float16,float16,0,0.029275199770927428
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,12,4,128,1,float16,float16,0,0.054416000843048096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,12,4,128,1,float16,float16,0,0.01918880045413971
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,12,2,128,1,float16,float16,0,0.01915999948978424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,12,12,128,1,float16,float16,0,0.01886720061302185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,12,1,128,1,float16,float16,0,0.015451200306415558
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,12,2,128,1,float16,float16,0,0.015758399665355683
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,12,4,128,1,float16,float16,0,0.01562879979610443
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,12,1,128,1,float16,float16,0,0.01555359959602356
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,12,12,128,1,float16,float16,0,0.016076800227165223
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,12,1,128,1,float16,float16,0,0.015118399262428283
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,12,2,128,1,float16,float16,0,0.01515199989080429
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,12,4,128,1,float16,float16,0,0.015148800611495972
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,12,1,128,1,float16,float16,0,0.014873600006103516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,12,12,128,1,float16,float16,0,0.01549919992685318
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,12,4,128,1,float16,float16,0,0.014841599762439728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,12,12,128,1,float16,float16,0,0.015105600655078887
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,12,1,128,1,float16,float16,0,0.014815999567508698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,12,2,128,1,float16,float16,0,0.014899200201034546
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,12,4,128,1,float16,float16,0,0.0147024005651474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,12,12,128,1,float16,float16,0,0.014867199957370758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,12,1,128,1,float16,float16,0,0.01483680009841919
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,12,2,128,1,float16,float16,0,0.014864000678062438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,12,4,128,1,float16,float16,0,0.014472000300884247
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,12,1,128,1,float16,float16,0,0.045444801449775696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,12,2,128,1,float16,float16,0,0.049451199173927304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,12,4,128,1,float16,float16,0,0.0692143976688385
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,12,12,128,1,float16,float16,0,0.08568959832191467
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,12,1,128,1,float16,float16,0,0.02754719853401184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,12,2,128,1,float16,float16,0,0.027803200483322143
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,12,4,128,1,float16,float16,0,0.03595199882984161
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,12,12,128,1,float16,float16,0,0.0443583995103836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,12,1,128,1,float16,float16,0,0.01934559941291809
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,12,2,128,1,float16,float16,0,0.019228799641132353
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,12,4,128,1,float16,float16,0,0.025444799661636354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,12,12,128,1,float16,float16,0,0.027046400308609008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,12,1,128,1,float16,float16,0,0.015911999344825744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,12,2,128,1,float16,float16,0,0.015887999534606935
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,12,4,128,1,float16,float16,0,0.018068799376487733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,12,12,128,1,float16,float16,0,0.017694400250911714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,12,1,128,1,float16,float16,0,0.014886400103569031
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,12,2,128,1,float16,float16,0,0.014838400483131408
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,12,4,128,1,float16,float16,0,0.01515520066022873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,12,12,128,1,float16,float16,0,0.015422399342060088
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,12,1,128,1,float16,float16,0,0.014686399698257446
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,12,2,128,1,float16,float16,0,0.014510400593280792
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,12,4,128,1,float16,float16,0,0.014569599926471711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,12,12,128,1,float16,float16,0,0.015041600167751312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,12,1,128,1,float16,float16,0,0.014585599303245544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,12,2,128,1,float16,float16,0,0.014407999813556671
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,12,4,128,1,float16,float16,0,0.014451199769973755
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,12,1,128,1,float16,float16,0,0.014582400023937226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,12,12,128,1,float16,float16,0,0.014763200283050537
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,12,4,128,1,float16,float16,0,0.014169600605964661
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,12,12,128,1,float16,float16,0,0.014721600711345673
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,12,1,128,1,float16,float16,0,0.014643199741840363
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,12,4,128,1,float16,float16,0,0.014235199987888336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,12,1,128,1,float16,float16,0,0.02624320089817047
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,12,2,128,1,float16,float16,0,0.03398720026016235
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,12,4,128,1,float16,float16,0,0.05146560072898865
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,12,12,128,1,float16,float16,0,0.06928960084915162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,12,1,128,1,float16,float16,0,0.018624000251293182
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,12,2,128,1,float16,float16,0,0.02451840043067932
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,12,4,128,1,float16,float16,0,0.03208000063896179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,12,12,128,1,float16,float16,0,0.041094401478767396
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,12,4,128,1,float16,float16,0,0.014803199470043183
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,12,2,128,1,float16,float16,0,0.01754080057144165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,12,4,128,1,float16,float16,0,0.02324319928884506
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,12,12,128,1,float16,float16,0,0.02543199956417084
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,12,1,128,1,float16,float16,0,0.014591999351978302
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,12,2,128,1,float16,float16,0,0.014812800288200378
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,12,4,128,1,float16,float16,0,0.016704000532627106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,12,12,128,1,float16,float16,0,0.016820800304412842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,12,1,128,1,float16,float16,0,0.013916799426078796
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,12,2,128,1,float16,float16,0,0.014177599549293518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,12,4,128,1,float16,float16,0,0.014283199608325959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,12,12,128,1,float16,float16,0,0.014996799826622009
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,12,1,128,1,float16,float16,0,0.013825599849224091
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,12,1,128,1,float16,float16,0,0.017367999255657195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,12,2,128,1,float16,float16,0,0.014043200016021728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,12,4,128,1,float16,float16,0,0.014091199636459351
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,12,1,128,1,float16,float16,0,0.01385439932346344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,12,2,128,1,float16,float16,0,0.013948799669742584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,12,4,128,1,float16,float16,0,0.014075200259685516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,12,12,128,1,float16,float16,0,0.014691199362277984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,12,1,128,1,float16,float16,0,0.013846400380134582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,12,2,128,1,float16,float16,0,0.013896000385284425
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,12,2,128,1,float16,float16,0,0.01478399932384491
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,12,4,128,1,float16,float16,0,0.013976000249385834
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,12,2,128,1,float16,float16,0,0.01401119977235794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,12,4,128,1,float16,float16,0,0.013967999815940857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,12,2,128,1,float16,float16,0,0.014550399780273438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,12,2,128,1,float16,float16,0,0.014452800154685974
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,12,1,128,1,float16,float16,0,0.01565759927034378
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,12,12,128,1,float16,float16,0,0.014975999295711518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,12,12,128,1,float16,float16,0,0.014468799531459808
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,12,1,128,1,float16,float16,0,0.014023999869823455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,8,1,128,1,float16,float16,0,10.57749252319336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,8,8,128,1,float16,float16,0,11.11810531616211
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,8,2,128,1,float16,float16,0,10.948343658447266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,8,4,128,1,float16,float16,0,10.998442840576171
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,8,1,128,1,float16,float16,0,5.443308639526367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,8,2,128,1,float16,float16,0,5.490367889404297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,8,8,128,1,float16,float16,0,5.80852165222168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,8,4,128,1,float16,float16,0,5.842567825317383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,8,1,128,1,float16,float16,0,21.26146545410156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,8,2,128,1,float16,float16,0,21.281532287597656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,8,8,128,1,float16,float16,0,3.153633689880371
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,8,1,128,1,float16,float16,0,2.864505577087402
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,8,2,128,1,float16,float16,0,2.955297660827637
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,8,4,128,1,float16,float16,0,3.088609504699707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,8,4,128,1,float16,float16,0,21.747744750976562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,8,8,128,1,float16,float16,0,3.4231632232666014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,8,1,128,1,float16,float16,0,6.1110481262207035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,8,8,128,1,float16,float16,0,6.4727935791015625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,8,2,128,1,float16,float16,0,6.116729736328125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,8,4,128,1,float16,float16,0,6.290431976318359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,8,1,128,1,float16,float16,0,3.2098880767822267
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,8,8,128,1,float16,float16,0,1.8752464294433593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,8,1,128,1,float16,float16,0,1.7265039443969727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,8,2,128,1,float16,float16,0,3.2456993103027343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,8,2,128,1,float16,float16,0,1.8003583908081056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,8,1,128,1,float16,float16,0,12.231646728515624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,8,4,128,1,float16,float16,0,3.3190799713134767
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,8,4,128,1,float16,float16,0,1.8279264450073243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,8,2,128,1,float16,float16,0,12.328577423095703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,8,4,128,1,float16,float16,0,12.40103988647461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,8,1,128,1,float16,float16,0,4.295830535888672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,8,8,128,1,float16,float16,0,2.4432735443115234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,8,8,128,1,float16,float16,0,4.642782211303711
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,8,2,128,1,float16,float16,0,4.325755310058594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,8,4,128,1,float16,float16,0,4.454841613769531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,8,8,128,1,float16,float16,0,1.363156795501709
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,8,1,128,1,float16,float16,0,1.2812159538269043
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,8,1,128,1,float16,float16,0,2.27748966217041
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,8,2,128,1,float16,float16,0,2.3457359313964843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,8,4,128,1,float16,float16,0,2.397105598449707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,8,1,128,1,float16,float16,0,8.420321655273437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,8,2,128,1,float16,float16,0,1.2415103912353516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,8,2,128,1,float16,float16,0,8.507640075683593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,8,4,128,1,float16,float16,0,1.3453104019165039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,8,4,128,1,float16,float16,0,8.82579345703125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,8,8,128,1,float16,float16,0,3.064297676086426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,8,1,128,1,float16,float16,0,5.369704055786133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,8,8,128,1,float16,float16,0,5.838164901733398
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,8,2,128,1,float16,float16,0,5.517812728881836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,8,4,128,1,float16,float16,0,5.667844772338867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,8,1,128,1,float16,float16,0,2.821544075012207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,8,8,128,1,float16,float16,0,1.6199968338012696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,8,1,128,1,float16,float16,0,1.5241920471191406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,8,2,128,1,float16,float16,0,2.887104034423828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,8,2,128,1,float16,float16,0,1.5946800231933593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,8,4,128,1,float16,float16,0,2.908076858520508
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,8,1,128,1,float16,float16,0,10.8337890625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,8,2,128,1,float16,float16,0,0.8767680168151856
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,8,1,128,1,float16,float16,0,0.8834287643432617
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,8,4,128,1,float16,float16,0,0.900051212310791
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,8,8,128,1,float16,float16,0,0.9386879920959472
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,8,2,128,1,float16,float16,0,11.003092956542968
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,8,4,128,1,float16,float16,0,1.5562895774841308
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,8,4,128,1,float16,float16,0,11.319735717773437
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,8,8,128,1,float16,float16,0,1.8140687942504883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,8,1,128,1,float16,float16,0,3.125695991516113
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,8,8,128,1,float16,float16,0,3.392225646972656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,8,2,128,1,float16,float16,0,3.1868127822875976
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,8,4,128,1,float16,float16,0,3.2176895141601562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,8,1,128,1,float16,float16,0,1.6680688858032227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,8,8,128,1,float16,float16,0,1.0093903541564941
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,8,1,128,1,float16,float16,0,0.9180656433105469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,8,2,128,1,float16,float16,0,1.7052448272705079
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,8,4,128,1,float16,float16,0,1.7135984420776367
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,8,2,128,1,float16,float16,0,0.9770832061767578
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,8,1,128,1,float16,float16,0,0.5570640087127685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,8,4,128,1,float16,float16,0,0.9552176475524903
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,8,1,128,1,float16,float16,0,6.19286880493164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,8,2,128,1,float16,float16,0,6.293588638305664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,8,4,128,1,float16,float16,0,6.281545639038086
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,8,4,128,1,float16,float16,0,0.5694320201873779
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,8,1,128,1,float16,float16,0,2.7991920471191407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,8,2,128,1,float16,float16,0,2.852324867248535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,8,8,128,1,float16,float16,0,0.6021647930145264
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,8,8,128,1,float16,float16,0,3.200494384765625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,8,2,128,1,float16,float16,0,0.5728975772857666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,8,1,128,1,float16,float16,0,1.430515193939209
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,8,2,128,1,float16,float16,0,1.5231696128845216
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,8,8,128,1,float16,float16,0,1.6901727676391602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,8,4,128,1,float16,float16,0,1.4989775657653808
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,8,1,128,1,float16,float16,0,5.663552093505859
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,8,2,128,1,float16,float16,0,5.684139251708984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,8,4,128,1,float16,float16,0,5.8383136749267575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,8,8,128,1,float16,float16,0,0.8829968452453614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,8,8,128,1,float16,float16,0,0.5560575962066651
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,8,1,128,1,float16,float16,0,0.8080816268920898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,8,4,128,1,float16,float16,0,2.916912078857422
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,8,1,128,1,float16,float16,0,0.48210558891296384
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,8,2,128,1,float16,float16,0,0.825715160369873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,8,4,128,1,float16,float16,0,0.8350031852722168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,8,2,128,1,float16,float16,0,0.49415998458862304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,8,4,128,1,float16,float16,0,0.4768511772155762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,8,8,128,1,float16,float16,0,0.3263040065765381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,8,2,128,1,float16,float16,0,0.3279167890548706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,8,1,128,1,float16,float16,0,0.3227695941925049
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,8,1,128,1,float16,float16,0,1.6210256576538087
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,8,8,128,1,float16,float16,0,1.8811552047729492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,8,1,128,1,float16,float16,0,3.1915184020996095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,8,8,128,1,float16,float16,0,1.0140192031860351
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,8,2,128,1,float16,float16,0,3.2331390380859375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,8,4,128,1,float16,float16,0,3.3523040771484376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,8,4,128,1,float16,float16,0,0.31543359756469724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,8,4,128,1,float16,float16,0,1.722870445251465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,8,1,128,1,float16,float16,0,0.9011008262634277
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,8,2,128,1,float16,float16,0,0.8856191635131836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,8,8,128,1,float16,float16,0,0.5814720153808594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,8,1,128,1,float16,float16,0,0.4916975975036621
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,8,4,128,1,float16,float16,0,0.9345088005065918
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,8,2,128,1,float16,float16,0,0.5095871925354004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,8,1,128,1,float16,float16,0,0.30757119655609133
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,8,2,128,1,float16,float16,0,0.29987359046936035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,8,8,128,1,float16,float16,0,0.3423903942108154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,8,4,128,1,float16,float16,0,0.5337168216705322
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,8,8,128,1,float16,float16,0,0.1939792037010193
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,8,4,128,1,float16,float16,0,0.31560161113739016
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,8,1,128,1,float16,float16,0,0.17044960260391234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,8,2,128,1,float16,float16,0,0.20301759243011475
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,8,2,128,1,float16,float16,0,1.6259231567382812
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,8,1,128,1,float16,float16,0,1.4584303855895997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,8,2,128,1,float16,float16,0,1.4920047760009765
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,8,8,128,1,float16,float16,0,1.7983375549316407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,8,1,128,1,float16,float16,0,2.889993667602539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,8,4,128,1,float16,float16,0,0.1974735975265503
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,8,2,128,1,float16,float16,0,2.9802751541137695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,8,8,128,1,float16,float16,0,0.9302255630493164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,8,4,128,1,float16,float16,0,3.1544687271118166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,8,1,128,1,float16,float16,0,0.783462381362915
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,8,4,128,1,float16,float16,0,1.6104848861694336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,8,8,128,1,float16,float16,0,0.5299183845520019
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,8,1,128,1,float16,float16,0,0.4282544136047363
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,8,2,128,1,float16,float16,0,0.8101823806762696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,8,8,128,1,float16,float16,0,0.29477438926696775
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,8,2,128,1,float16,float16,0,0.461030387878418
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,8,4,128,1,float16,float16,0,0.8185903549194335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,8,1,128,1,float16,float16,0,0.25887839794158934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,8,4,128,1,float16,float16,0,0.2664351940155029
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,8,2,128,1,float16,float16,0,0.26082720756530764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,8,8,128,1,float16,float16,0,0.18029279708862306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,8,2,128,1,float16,float16,0,0.17418719530105592
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,8,1,128,1,float16,float16,0,0.16765120029449462
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,8,8,128,1,float16,float16,0,0.13612159490585327
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,8,1,128,1,float16,float16,0,0.1023792028427124
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,8,4,128,1,float16,float16,0,0.16817920207977294
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,8,2,128,1,float16,float16,0,0.11157599687576295
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,8,4,128,1,float16,float16,0,0.12886079549789428
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,8,1,128,1,float16,float16,0,0.8643551826477051
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,8,2,128,1,float16,float16,0,0.9068479537963867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,8,4,128,1,float16,float16,0,0.9530991554260254
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,8,8,128,1,float16,float16,0,1.1299087524414062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,8,1,128,1,float16,float16,0,1.7024688720703125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,8,8,128,1,float16,float16,0,0.586513614654541
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,8,2,128,1,float16,float16,0,1.736180877685547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,8,1,128,1,float16,float16,0,0.47203521728515624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,8,2,128,1,float16,float16,0,0.48512802124023435
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,8,4,128,1,float16,float16,0,1.8744112014770509
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,8,4,128,1,float16,float16,0,0.5135744094848633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,8,8,128,1,float16,float16,0,0.34183359146118164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,8,4,128,1,float16,float16,0,0.45282878875732424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,8,4,128,1,float16,float16,0,0.28714239597320557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,8,2,128,1,float16,float16,0,0.2760576009750366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,8,8,128,1,float16,float16,0,0.12941919565200805
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,8,1,128,1,float16,float16,0,0.15886399745941163
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,8,2,128,1,float16,float16,0,0.10579040050506591
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,8,4,128,1,float16,float16,0,0.09697920083999634
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,8,8,128,1,float16,float16,0,0.07722240090370178
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,8,1,128,1,float16,float16,0,0.07296800017356872
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,8,2,128,1,float16,float16,0,0.07258719801902772
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,8,4,128,1,float16,float16,0,0.07418559789657593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,8,1,128,1,float16,float16,0,2.319718360900879
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,8,1,128,1,float16,float16,0,0.2683056116104126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,8,2,128,1,float16,float16,0,2.423521614074707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,8,4,128,1,float16,float16,0,0.17891839742660523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,8,4,128,1,float16,float16,0,2.647902488708496
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,8,1,128,1,float16,float16,0,0.10891200304031372
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,8,1,128,1,float16,float16,0,1.164303970336914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,8,1,128,1,float16,float16,0,0.6020480155944824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,8,8,128,1,float16,float16,0,1.5574480056762696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,8,2,128,1,float16,float16,0,1.2272768020629883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,8,4,128,1,float16,float16,0,1.317296028137207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,8,2,128,1,float16,float16,0,0.6294608116149902
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,8,4,128,1,float16,float16,0,0.678985595703125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,8,8,128,1,float16,float16,0,0.40929598808288575
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,8,1,128,1,float16,float16,0,0.315228796005249
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,8,2,128,1,float16,float16,0,0.33313119411468506
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,8,1,128,1,float16,float16,0,0.1736400008201599
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,8,8,128,1,float16,float16,0,0.22099199295043945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,8,4,128,1,float16,float16,0,0.354203200340271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,8,2,128,1,float16,float16,0,0.18132959604263305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,8,4,128,1,float16,float16,0,0.19445120096206664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,8,8,128,1,float16,float16,0,0.12564480304718018
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,8,2,128,1,float16,float16,0,0.17300000190734863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,8,1,128,1,float16,float16,0,0.10365760326385498
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,8,4,128,1,float16,float16,0,0.11505440473556519
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,8,8,128,1,float16,float16,0,0.07551360130310059
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,8,1,128,1,float16,float16,0,0.06476320028305053
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,8,2,128,1,float16,float16,0,0.06545119881629943
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,8,4,128,1,float16,float16,0,0.06748480200767518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,8,8,128,1,float16,float16,0,0.06916959881782532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,8,1,128,1,float16,float16,0,0.05399680137634277
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,8,2,128,1,float16,float16,0,0.053595197200775144
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,8,4,128,1,float16,float16,0,0.06568319797515869
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,8,8,128,1,float16,float16,0,0.7946080207824707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,8,1,128,1,float16,float16,0,0.8035872459411622
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,8,2,128,1,float16,float16,0,0.1072543978691101
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,8,8,128,1,float16,float16,0,1.2002991676330566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,8,1,128,1,float16,float16,0,1.572657585144043
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,8,2,128,1,float16,float16,0,1.6867279052734374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,8,4,128,1,float16,float16,0,1.867620849609375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,8,2,128,1,float16,float16,0,0.8603648185729981
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,8,1,128,1,float16,float16,0,0.41561121940612794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,8,2,128,1,float16,float16,0,0.44282240867614747
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,8,8,128,1,float16,float16,0,0.6136096000671387
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,8,4,128,1,float16,float16,0,0.940833568572998
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,8,4,128,1,float16,float16,0,0.48583040237426756
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,8,8,128,1,float16,float16,0,0.3198224067687988
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,8,1,128,1,float16,float16,0,0.2216480016708374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,8,2,128,1,float16,float16,0,0.23538880348205565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,8,8,128,1,float16,float16,0,0.17231040000915526
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,8,1,128,1,float16,float16,0,0.12353919744491577
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,8,4,128,1,float16,float16,0,0.25715041160583496
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,8,2,128,1,float16,float16,0,0.13127520084381103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,8,8,128,1,float16,float16,0,0.0991487979888916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,8,4,128,1,float16,float16,0,0.14326239824295045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,8,2,128,1,float16,float16,0,0.0771727979183197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,8,1,128,1,float16,float16,0,0.0736303985118866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,8,1,128,1,float16,float16,0,0.04421440064907074
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,8,8,128,1,float16,float16,0,0.06405599713325501
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,8,8,128,1,float16,float16,0,0.06070240139961243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,8,2,128,1,float16,float16,0,0.04469760060310364
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,8,4,128,1,float16,float16,0,0.04621120095252991
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,8,1,128,1,float16,float16,0,0.048785600066185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,8,2,128,1,float16,float16,0,0.05280159711837769
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,8,8,128,1,float16,float16,0,0.03655200004577637
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,8,4,128,1,float16,float16,0,0.054011201858520506
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,8,1,128,1,float16,float16,0,0.03529120087623596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,8,2,128,1,float16,float16,0,0.035648000240325925
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,8,4,128,1,float16,float16,0,0.035812801122665404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,8,8,128,1,float16,float16,0,0.19601600170135497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,8,2,128,1,float16,float16,0,0.40409440994262696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,8,1,128,1,float16,float16,0,0.19294079542160034
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,8,8,128,1,float16,float16,0,0.3907583951950073
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,8,1,128,1,float16,float16,0,0.36133599281311035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,8,4,128,1,float16,float16,0,0.2584160089492798
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,8,8,128,1,float16,float16,0,0.2096031904220581
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,8,1,128,1,float16,float16,0,0.11532800197601319
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,8,4,128,1,float16,float16,0,0.504092788696289
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,8,4,128,1,float16,float16,0,0.14390560388565063
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,8,2,128,1,float16,float16,0,0.12049280405044556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,8,2,128,1,float16,float16,0,0.07217119932174683
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,8,1,128,1,float16,float16,0,0.06675840020179749
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,8,1,128,1,float16,float16,0,0.039526399970054624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,8,4,128,1,float16,float16,0,0.08261600136756897
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,8,2,128,1,float16,float16,0,0.04244639873504639
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,8,4,128,1,float16,float16,0,0.04877600073814392
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,8,8,128,1,float16,float16,0,0.03781279921531677
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,8,4,128,1,float16,float16,0,0.032380801439285276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,8,1,128,1,float16,float16,0,0.02871679961681366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,8,8,128,1,float16,float16,0,0.025838398933410646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,8,1,128,1,float16,float16,0,0.02317280024290085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,8,4,128,1,float16,float16,0,0.02388159930706024
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,8,2,128,1,float16,float16,0,0.02326720058917999
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,8,8,128,1,float16,float16,0,0.024292799830436706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,8,1,128,1,float16,float16,0,0.022891199588775633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,8,4,128,1,float16,float16,0,0.022907200455665588
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,8,2,128,1,float16,float16,0,0.022752000391483305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,8,8,128,1,float16,float16,0,0.02396160066127777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,8,1,128,1,float16,float16,0,0.02253919988870621
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,8,2,128,1,float16,float16,0,0.02248319983482361
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,8,4,128,1,float16,float16,0,0.02298240065574646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,8,1,128,1,float16,float16,0,0.16136159896850585
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,8,2,128,1,float16,float16,0,0.18734400272369384
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,8,4,128,1,float16,float16,0,0.2360703945159912
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,8,1,128,1,float16,float16,0,0.09109600186347962
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,8,4,128,1,float16,float16,0,0.08351200222969055
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,8,2,128,1,float16,float16,0,0.10519360303878784
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,8,4,128,1,float16,float16,0,0.13048479557037354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,8,8,128,1,float16,float16,0,0.1114799976348877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,8,1,128,1,float16,float16,0,0.05691360235214234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,8,2,128,1,float16,float16,0,0.061552000045776364
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,8,4,128,1,float16,float16,0,0.07294880151748658
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,8,8,128,1,float16,float16,0,0.060022401809692386
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,8,2,128,1,float16,float16,0,0.03267520070075989
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,8,1,128,1,float16,float16,0,0.03102880120277405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,8,4,128,1,float16,float16,0,0.04301599860191345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,8,8,128,1,float16,float16,0,0.029185599088668822
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,8,1,128,1,float16,float16,0,0.02170879989862442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,8,2,128,1,float16,float16,0,0.023515200614929198
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,8,4,128,1,float16,float16,0,0.024212799966335297
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,8,8,128,1,float16,float16,0,0.020819200575351714
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,8,1,128,1,float16,float16,0,0.018160000443458557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,8,2,128,1,float16,float16,0,0.018641600012779237
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,8,4,128,1,float16,float16,0,0.018675200641155243
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,8,8,128,1,float16,float16,0,0.019153599441051484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,8,1,128,1,float16,float16,0,0.017774400115013123
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,8,2,128,1,float16,float16,0,0.21513919830322265
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,8,4,128,1,float16,float16,0,0.017953599989414214
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,8,1,128,1,float16,float16,0,0.017481599748134614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,8,8,128,1,float16,float16,0,0.018492799997329713
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,8,8,128,1,float16,float16,0,0.11771520376205444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,8,2,128,1,float16,float16,0,0.01751679927110672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,8,4,128,1,float16,float16,0,0.017671999335289002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,8,1,128,1,float16,float16,0,0.017257599532604216
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,8,2,128,1,float16,float16,0,0.017209599912166595
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,8,8,128,1,float16,float16,0,0.06552479863166809
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,8,4,128,1,float16,float16,0,0.017319999635219574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,8,2,128,1,float16,float16,0,0.0282368004322052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,8,1,128,1,float16,float16,0,0.08892639875411987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,8,2,128,1,float16,float16,0,0.10486719608306885
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,8,4,128,1,float16,float16,0,0.1293455958366394
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,8,1,128,1,float16,float16,0,0.0542639970779419
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,8,2,128,1,float16,float16,0,0.06050879955291748
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,8,4,128,1,float16,float16,0,0.07410079836845399
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,8,8,128,1,float16,float16,0,0.060059201717376706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,8,1,128,1,float16,float16,0,0.029150399565696716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,8,2,128,1,float16,float16,0,0.030934399366378783
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,8,4,128,1,float16,float16,0,0.041555199027061465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,8,8,128,1,float16,float16,0,0.030112001299858093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,8,1,128,1,float16,float16,0,0.019435200095176696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,8,4,128,1,float16,float16,0,0.020895999670028687
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,8,8,128,1,float16,float16,0,0.019246399402618408
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,8,1,128,1,float16,float16,0,0.015878400206565856
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,8,2,128,1,float16,float16,0,0.01589120030403137
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,8,4,128,1,float16,float16,0,0.016519999504089354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,8,8,128,1,float16,float16,0,0.01589920073747635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,8,1,128,1,float16,float16,0,0.015384000539779664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,8,2,128,1,float16,float16,0,0.015337599813938141
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,8,8,128,1,float16,float16,0,0.1959007978439331
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,8,4,128,1,float16,float16,0,0.015609599649906158
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,8,8,128,1,float16,float16,0,0.015203200280666351
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,8,2,128,1,float16,float16,0,0.015115199983119965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,8,4,128,1,float16,float16,0,0.015268799662590028
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,8,1,128,1,float16,float16,0,0.015156799554824829
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,8,8,128,1,float16,float16,0,0.014849600195884705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,8,1,128,1,float16,float16,0,0.014870400726795196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,8,2,128,1,float16,float16,0,0.014822399616241455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,8,4,128,1,float16,float16,0,0.014867199957370758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,8,8,128,1,float16,float16,0,0.014588800072669984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,8,1,128,1,float16,float16,0,0.014975999295711518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,8,2,128,1,float16,float16,0,0.014913600683212281
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,8,4,128,1,float16,float16,0,0.014804799854755402
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,8,1,128,1,float16,float16,0,0.05584959983825684
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,8,2,128,1,float16,float16,0,0.061761599779129026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,8,4,128,1,float16,float16,0,0.08293759822845459
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,8,1,128,1,float16,float16,0,0.029905599355697633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,8,8,128,1,float16,float16,0,0.07414559721946716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,8,2,128,1,float16,float16,0,0.031044799089431762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,8,8,128,1,float16,float16,0,0.03857919871807099
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,8,4,128,1,float16,float16,0,0.04831359982490539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,8,1,128,1,float16,float16,0,0.01987359970808029
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,8,2,128,1,float16,float16,0,0.020175999402999877
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,8,4,128,1,float16,float16,0,0.026819199323654175
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,8,8,128,1,float16,float16,0,0.02553279995918274
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,8,1,128,1,float16,float16,0,0.016318400204181672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,8,2,128,1,float16,float16,0,0.017985600233078002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,8,2,128,1,float16,float16,0,0.016548800468444824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,8,4,128,1,float16,float16,0,0.01854880005121231
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,8,8,128,1,float16,float16,0,0.01770240068435669
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,8,1,128,1,float16,float16,0,0.015304000675678253
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,8,4,128,1,float16,float16,0,0.015323199331760406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,8,8,128,1,float16,float16,0,0.015281599760055543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,8,1,128,1,float16,float16,0,0.015083199739456177
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,8,8,128,1,float16,float16,0,0.01831520050764084
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,8,8,128,1,float16,float16,0,0.014905600249767304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,8,1,128,1,float16,float16,0,0.014847999811172486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,8,2,128,1,float16,float16,0,0.015004800260066986
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,8,4,128,1,float16,float16,0,0.014588800072669984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,8,8,128,1,float16,float16,0,0.01467519998550415
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,8,2,128,1,float16,float16,0,0.014883199334144592
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,8,1,128,1,float16,float16,0,0.014703999459743499
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,8,4,128,1,float16,float16,0,0.014537599682807923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,8,8,128,1,float16,float16,0,0.1085919976234436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,8,8,128,1,float16,float16,0,0.014347200095653535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,8,1,128,1,float16,float16,0,0.014921599626541137
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,8,4,128,1,float16,float16,0,0.014441600441932679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,8,1,128,1,float16,float16,0,0.03210720121860504
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,8,2,128,1,float16,float16,0,0.03912639915943146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,8,2,128,1,float16,float16,0,0.019886399805545806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,8,4,128,1,float16,float16,0,0.06563519835472106
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,8,8,128,1,float16,float16,0,0.05675519704818725
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,8,1,128,1,float16,float16,0,0.020904000103473663
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,8,2,128,1,float16,float16,0,0.0267551988363266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,8,4,128,1,float16,float16,0,0.03480800092220306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,8,8,128,1,float16,float16,0,0.033478400111198424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,8,1,128,1,float16,float16,0,0.017081600427627564
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,8,2,128,1,float16,float16,0,0.01871519982814789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,8,4,128,1,float16,float16,0,0.02498079985380173
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,8,8,128,1,float16,float16,0,0.02402399927377701
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,8,1,128,1,float16,float16,0,0.015835200250148774
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,8,4,128,1,float16,float16,0,0.017748799920082093
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,8,8,128,1,float16,float16,0,0.01693120002746582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,8,1,128,1,float16,float16,0,0.014847999811172486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,8,2,128,1,float16,float16,0,0.014617599546909332
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,8,4,128,1,float16,float16,0,0.014865599572658539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,8,8,128,1,float16,float16,0,0.014798399806022645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,8,1,128,1,float16,float16,0,0.014638400077819825
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,8,2,128,1,float16,float16,0,0.014292800426483154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,8,4,128,1,float16,float16,0,0.014667199552059173
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,8,8,128,1,float16,float16,0,0.014545600116252898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,8,1,128,1,float16,float16,0,0.014788800477981567
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,8,4,128,1,float16,float16,0,0.014382399618625641
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,8,8,128,1,float16,float16,0,0.014347200095653535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,8,2,128,1,float16,float16,0,0.015425600111484528
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,8,1,128,1,float16,float16,0,0.014740799367427827
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,8,2,128,1,float16,float16,0,0.014369599521160126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,8,4,128,1,float16,float16,0,0.014276799559593201
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,8,8,128,1,float16,float16,0,0.014407999813556671
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,8,1,128,1,float16,float16,0,0.014584000408649444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,8,2,128,1,float16,float16,0,0.014905600249767304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,8,4,128,1,float16,float16,0,0.014051200449466705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,8,1,128,1,float16,float16,0,0.02619200050830841
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,8,2,128,1,float16,float16,0,0.03377600014209747
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,8,4,128,1,float16,float16,0,0.05152159929275513
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,8,8,128,1,float16,float16,0,0.04965600073337555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,8,1,128,1,float16,float16,0,0.018568000197410582
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,8,2,128,1,float16,float16,0,0.024190400540828706
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,8,4,128,1,float16,float16,0,0.03213759958744049
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,8,8,128,1,float16,float16,0,0.030875200033187868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,8,1,128,1,float16,float16,0,0.01568000018596649
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,8,2,128,1,float16,float16,0,0.014638400077819825
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,8,2,128,1,float16,float16,0,0.017398400604724883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,8,4,128,1,float16,float16,0,0.02338559925556183
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,8,8,128,1,float16,float16,0,0.022598400712013245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,8,1,128,1,float16,float16,0,0.014672000706195832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,8,2,128,1,float16,float16,0,0.01499519944190979
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,8,4,128,1,float16,float16,0,0.01675360053777695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,8,8,128,1,float16,float16,0,0.016065600514411926
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,8,1,128,1,float16,float16,0,0.014155200123786927
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,8,2,128,1,float16,float16,0,0.014054399728775025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,8,4,128,1,float16,float16,0,0.014251199364662171
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,8,8,128,1,float16,float16,0,0.014256000518798828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,8,1,128,1,float16,float16,0,0.013804799318313599
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,8,2,128,1,float16,float16,0,0.014070400595664978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,8,4,128,1,float16,float16,0,0.014164799451828003
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,8,8,128,1,float16,float16,0,0.014256000518798828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,8,1,128,1,float16,float16,0,0.01395840048789978
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,8,2,128,1,float16,float16,0,0.013822400569915771
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,8,4,128,1,float16,float16,0,0.014000000059604644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,8,8,128,1,float16,float16,0,0.014115199446678162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,8,1,128,1,float16,float16,0,0.013961599767208099
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,8,2,128,1,float16,float16,0,0.013748799264431
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,8,4,128,1,float16,float16,0,0.014046399295330048
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,8,8,128,1,float16,float16,0,0.013857600092887879
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,8,2,128,1,float16,float16,0,0.01557919979095459
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,8,1,128,1,float16,float16,0,0.013784000277519226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,8,2,128,1,float16,float16,0,0.014027200639247894
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,8,4,128,1,float16,float16,0,0.013867199420928955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,8,2,128,1,float16,float16,0,0.0144896000623703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,8,2,128,1,float16,float16,0,0.01440960019826889
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,4,4,128,1,float16,float16,0,3.110641670227051
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,4,1,128,1,float16,float16,0,5.5970207214355465
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,8,4,128,1,float16,float16,0,0.014956800639629364
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,4,4,128,1,float16,float16,0,5.848734283447266
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,4,2,128,1,float16,float16,0,5.6554302215576175
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,4,4,128,1,float16,float16,0,1.7447759628295898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,4,1,128,1,float16,float16,0,1.7528432846069335
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,4,1,128,1,float16,float16,0,2.959097671508789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,4,2,128,1,float16,float16,0,3.1399408340454102
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,4,2,128,1,float16,float16,0,1.7579679489135742
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,4,1,128,1,float16,float16,0,10.665033721923828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,4,2,128,1,float16,float16,0,11.1216796875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,4,4,128,1,float16,float16,0,1.8712896347045898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,4,1,128,1,float16,float16,0,3.238782501220703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,4,4,128,1,float16,float16,0,3.3063278198242188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,4,4,128,1,float16,float16,0,1.0698335647583008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,4,2,128,1,float16,float16,0,3.3048351287841795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,4,1,128,1,float16,float16,0,1.7667312622070312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,4,1,128,1,float16,float16,0,1.0999360084533691
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,4,2,128,1,float16,float16,0,1.148140811920166
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,4,2,128,1,float16,float16,0,1.8894527435302735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,4,1,128,1,float16,float16,0,6.182537460327149
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,4,2,128,1,float16,float16,0,6.174683380126953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,4,4,128,1,float16,float16,0,1.381873607635498
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,4,4,128,1,float16,float16,0,2.373931121826172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,4,1,128,1,float16,float16,0,1.2976976394653321
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,4,1,128,1,float16,float16,0,2.3171920776367188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,4,2,128,1,float16,float16,0,2.3336320877075196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,4,4,128,1,float16,float16,0,0.8609840393066406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,4,2,128,1,float16,float16,0,1.358518409729004
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,4,1,128,1,float16,float16,0,0.8424511909484863
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,4,2,128,1,float16,float16,0,0.849947166442871
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,4,1,128,1,float16,float16,0,4.379131317138672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,4,2,128,1,float16,float16,0,4.498817443847656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,4,4,128,1,float16,float16,0,1.6039695739746094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,4,1,128,1,float16,float16,0,1.57882080078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,4,4,128,1,float16,float16,0,3.007953643798828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,4,2,128,1,float16,float16,0,1.5881664276123046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,4,4,128,1,float16,float16,0,0.9319919586181641
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,4,1,128,1,float16,float16,0,2.8334911346435545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,4,2,128,1,float16,float16,0,2.83917121887207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,4,1,128,1,float16,float16,0,0.8720303535461426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,4,4,128,1,float16,float16,0,0.591929578781128
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,4,1,128,1,float16,float16,0,0.6267119884490967
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,4,2,128,1,float16,float16,0,0.9061440467834473
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,4,2,128,1,float16,float16,0,0.6200240135192872
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,4,1,128,1,float16,float16,0,5.542222213745117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,4,2,128,1,float16,float16,0,5.603582382202148
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,4,1,128,1,float16,float16,0,1.6685264587402344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,4,2,128,1,float16,float16,0,1.672091293334961
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,4,4,128,1,float16,float16,0,1.7902687072753907
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,4,4,128,1,float16,float16,0,1.0202624320983886
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,4,1,128,1,float16,float16,0,0.9869232177734375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,4,4,128,1,float16,float16,0,0.6002480030059815
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,4,1,128,1,float16,float16,0,0.5736735820770263
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,4,2,128,1,float16,float16,0,0.9526592254638672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,4,2,128,1,float16,float16,0,0.5662479877471924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,4,4,128,1,float16,float16,0,0.39449760913848875
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,4,1,128,1,float16,float16,0,3.162265586853027
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,4,1,128,1,float16,float16,0,0.3220992088317871
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,4,2,128,1,float16,float16,0,0.3915584087371826
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,4,2,128,1,float16,float16,0,3.1905471801757814
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,4,4,128,1,float16,float16,0,0.8757216453552246
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,4,1,128,1,float16,float16,0,0.8378255844116211
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,4,1,128,1,float16,float16,0,1.488644790649414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,4,2,128,1,float16,float16,0,1.5011728286743165
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,4,4,128,1,float16,float16,0,1.6110048294067383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,4,4,128,1,float16,float16,0,0.5184991836547852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,4,2,128,1,float16,float16,0,0.8168095588684082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,4,1,128,1,float16,float16,0,0.4654128074645996
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,4,4,128,1,float16,float16,0,0.3295696020126343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,4,1,128,1,float16,float16,0,0.32084479331970217
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,4,2,128,1,float16,float16,0,0.479585599899292
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,4,2,128,1,float16,float16,0,0.30840320587158204
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,4,1,128,1,float16,float16,0,2.8607343673706054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,4,1,128,1,float16,float16,0,0.19130719900131227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,4,2,128,1,float16,float16,0,0.20623359680175782
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,4,2,128,1,float16,float16,0,2.9255632400512694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,4,1,128,1,float16,float16,0,0.8756496429443359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,4,4,128,1,float16,float16,0,0.5627391815185547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,4,2,128,1,float16,float16,0,0.8962032318115234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,4,4,128,1,float16,float16,0,1.0053135871887207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,4,1,128,1,float16,float16,0,1.6404895782470703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,4,1,128,1,float16,float16,0,0.49352641105651857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,4,4,128,1,float16,float16,0,0.33893918991088867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,4,2,128,1,float16,float16,0,0.5112480163574219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,4,2,128,1,float16,float16,0,1.6711999893188476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,4,1,128,1,float16,float16,0,0.31027679443359374
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,4,4,128,1,float16,float16,0,0.1916864037513733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,4,1,128,1,float16,float16,0,0.20360000133514405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,4,2,128,1,float16,float16,0,0.2986464023590088
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,4,2,128,1,float16,float16,0,0.19631359577178956
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,4,4,128,1,float16,float16,0,0.13770079612731934
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,4,2,128,1,float16,float16,0,0.13568639755249023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,4,1,128,1,float16,float16,0,0.13216480016708373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,4,1,128,1,float16,float16,0,0.536630392074585
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,4,4,128,1,float16,float16,0,0.6086304187774658
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,4,4,128,1,float16,float16,0,0.24642400741577147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,4,1,128,1,float16,float16,0,1.0160752296447755
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,4,2,128,1,float16,float16,0,1.075268840789795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,4,4,128,1,float16,float16,0,1.1597279548645019
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,4,4,128,1,float16,float16,0,0.32716319561004636
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,4,1,128,1,float16,float16,0,0.28902239799499513
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,4,2,128,1,float16,float16,0,0.5626224040985107
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,4,4,128,1,float16,float16,0,0.1913632035255432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,4,2,128,1,float16,float16,0,0.3042191982269287
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,4,4,128,1,float16,float16,0,0.11371519565582275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,4,1,128,1,float16,float16,0,0.17242079973220825
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,4,2,128,1,float16,float16,0,0.17971040010452272
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,4,1,128,1,float16,float16,0,1.9941728591918946
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,4,4,128,1,float16,float16,0,0.0864463984966278
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,4,1,128,1,float16,float16,0,0.10497440099716186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,4,1,128,1,float16,float16,0,0.07532479763031005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,4,2,128,1,float16,float16,0,2.0846704483032226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,4,2,128,1,float16,float16,0,0.07609919905662536
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,4,1,128,1,float16,float16,0,0.3371119976043701
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,4,4,128,1,float16,float16,0,0.39957759380340574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,4,1,128,1,float16,float16,0,0.6296512126922608
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,4,2,128,1,float16,float16,0,0.6673647880554199
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,4,4,128,1,float16,float16,0,0.22368481159210205
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,4,1,128,1,float16,float16,0,0.1915055990219116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,4,1,128,1,float16,float16,0,1.2141200065612794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,4,2,128,1,float16,float16,0,0.2010256052017212
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,4,4,128,1,float16,float16,0,0.13126399517059326
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,4,2,128,1,float16,float16,0,1.2896639823913574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,4,1,128,1,float16,float16,0,0.11575679779052735
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,4,4,128,1,float16,float16,0,0.08453919887542724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,4,2,128,1,float16,float16,0,0.1237023949623108
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,4,1,128,1,float16,float16,0,0.0824015974998474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,4,1,128,1,float16,float16,0,0.05905600190162659
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,4,4,128,1,float16,float16,0,0.06079999804496765
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,4,2,128,1,float16,float16,0,0.06023039817810059
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,4,1,128,1,float16,float16,0,0.6354976177215577
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,4,4,128,1,float16,float16,0,0.7866479873657226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,4,2,128,1,float16,float16,0,0.10671360492706299
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,4,1,128,1,float16,float16,0,1.2376111984252929
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,4,2,128,1,float16,float16,0,1.3327520370483399
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,4,1,128,1,float16,float16,0,0.3401648044586182
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,4,4,128,1,float16,float16,0,0.41637439727783204
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,4,2,128,1,float16,float16,0,0.3649264097213745
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,4,2,128,1,float16,float16,0,0.6891136169433594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,4,4,128,1,float16,float16,0,0.22884960174560548
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,4,1,128,1,float16,float16,0,0.18966079950332643
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,4,4,128,1,float16,float16,0,0.7404784202575684
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,4,2,128,1,float16,float16,0,0.20240321159362792
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,4,4,128,1,float16,float16,0,0.13408960103988649
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,4,1,128,1,float16,float16,0,0.11234879493713379
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,4,2,128,1,float16,float16,0,0.119377601146698
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,4,4,128,1,float16,float16,0,0.07630720138549804
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,4,4,128,1,float16,float16,0,0.06916959881782532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,4,1,128,1,float16,float16,0,0.05353760123252869
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,4,1,128,1,float16,float16,0,0.06552159786224365
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,4,2,128,1,float16,float16,0,0.06543999910354614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,4,2,128,1,float16,float16,0,0.08234400153160096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,4,4,128,1,float16,float16,0,0.044512000679969785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,4,1,128,1,float16,float16,0,0.043289598822593686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,4,2,128,1,float16,float16,0,0.04408159852027893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,4,1,128,1,float16,float16,0,0.4474143981933594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,4,1,128,1,float16,float16,0,0.8630800247192383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,4,4,128,1,float16,float16,0,0.6155871868133544
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,4,2,128,1,float16,float16,0,0.4968736171722412
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,4,2,128,1,float16,float16,0,0.35518879890441896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,4,2,128,1,float16,float16,0,0.06718720197677612
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,4,4,128,1,float16,float16,0,0.3233920097351074
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,4,1,128,1,float16,float16,0,0.13375840187072754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,4,1,128,1,float16,float16,0,0.23918559551239013
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,4,4,128,1,float16,float16,0,0.17613600492477416
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,4,2,128,1,float16,float16,0,0.2641551971435547
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,4,2,128,1,float16,float16,0,0.1450703978538513
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,4,4,128,1,float16,float16,0,0.10360959768295289
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,4,1,128,1,float16,float16,0,0.07963520288467407
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,4,2,128,1,float16,float16,0,0.08902879953384399
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,4,4,128,1,float16,float16,0,0.06809599995613098
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,4,1,128,1,float16,float16,0,0.05280799865722656
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,4,4,128,1,float16,float16,0,0.060031998157501223
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,4,2,128,1,float16,float16,0,0.05429440140724182
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,4,4,128,1,float16,float16,0,0.03641600012779236
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,4,1,128,1,float16,float16,0,0.035462400317192076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,4,2,128,1,float16,float16,0,0.05355839729309082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,4,4,128,1,float16,float16,0,0.02869119942188263
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,4,2,128,1,float16,float16,0,0.03553279936313629
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,4,1,128,1,float16,float16,0,0.028270399570465087
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,4,2,128,1,float16,float16,0,0.028681600093841554
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,4,1,128,1,float16,float16,0,0.11875360012054444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,4,2,128,1,float16,float16,0,0.26023681163787843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,4,4,128,1,float16,float16,0,0.20680639743804932
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,4,2,128,1,float16,float16,0,0.14495680332183838
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,4,1,128,1,float16,float16,0,0.21764800548553467
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,4,4,128,1,float16,float16,0,0.06532160043716431
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,4,2,128,1,float16,float16,0,0.08186720013618469
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,4,1,128,1,float16,float16,0,0.07187520265579224
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,4,1,128,1,float16,float16,0,0.04437919855117798
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,4,2,128,1,float16,float16,0,0.05018720030784607
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,4,4,128,1,float16,float16,0,0.037651199102401736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,4,1,128,1,float16,float16,0,0.02834559977054596
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,4,2,128,1,float16,float16,0,0.032323199510574344
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,4,4,128,1,float16,float16,0,0.025758400559425354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,4,1,128,1,float16,float16,0,0.023454399406909944
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,4,1,128,1,float16,float16,0,0.022920000553131103
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,4,2,128,1,float16,float16,0,0.022987200319766997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,4,1,128,1,float16,float16,0,0.02266719937324524
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,4,2,128,1,float16,float16,0,0.022700800001621245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,4,2,128,1,float16,float16,0,0.9675744056701661
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,4,2,128,1,float16,float16,0,0.02247679978609085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,4,1,128,1,float16,float16,0,0.1088479995727539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,4,2,128,1,float16,float16,0,0.13470079898834228
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,4,1,128,1,float16,float16,0,0.05206239819526672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,4,4,128,1,float16,float16,0,0.1146496057510376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,4,1,128,1,float16,float16,0,0.06266720294952392
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,4,2,128,1,float16,float16,0,0.07470719814300537
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,4,4,128,1,float16,float16,0,0.060171198844909665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,4,2,128,1,float16,float16,0,0.043819200992584226
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,4,1,128,1,float16,float16,0,0.03615840077400208
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,4,4,128,1,float16,float16,0,0.03009760081768036
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,4,1,128,1,float16,float16,0,0.02421119958162308
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,4,2,128,1,float16,float16,0,0.024937599897384644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,4,4,128,1,float16,float16,0,0.020839999616146087
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,4,1,128,1,float16,float16,0,0.018606400489807128
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,4,4,128,1,float16,float16,0,0.11771680116653442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,4,2,128,1,float16,float16,0,0.01902240067720413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,4,4,128,1,float16,float16,0,0.01914079934358597
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,4,1,128,1,float16,float16,0,0.018027199804782866
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,4,4,128,1,float16,float16,0,0.01860959976911545
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,4,1,128,1,float16,float16,0,0.017815999686717987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,4,2,128,1,float16,float16,0,0.017601600289344786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,4,4,128,1,float16,float16,0,0.024255999922752382
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,4,4,128,1,float16,float16,0,0.018328000605106354
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,4,1,128,1,float16,float16,0,0.01732639968395233
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,4,2,128,1,float16,float16,0,0.017360000312328337
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,4,4,128,1,float16,float16,0,0.023907199501991272
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,4,1,128,1,float16,float16,0,0.01720000058412552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,4,2,128,1,float16,float16,0,0.017203199863433837
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,4,4,128,1,float16,float16,0,0.023319999873638152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,4,2,128,1,float16,float16,0,0.0748528003692627
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,4,4,128,1,float16,float16,0,0.06131680011749267
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,4,1,128,1,float16,float16,0,0.03166239857673645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,4,4,128,1,float16,float16,0,0.03308640122413635
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,4,1,128,1,float16,float16,0,0.02043039947748184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,4,2,128,1,float16,float16,0,0.02152640074491501
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,4,4,128,1,float16,float16,0,0.019849599897861482
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,4,1,128,1,float16,float16,0,0.016702400147914888
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,4,2,128,1,float16,float16,0,0.016940799355506898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,4,4,128,1,float16,float16,0,0.015857599675655365
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,4,1,128,1,float16,float16,0,0.015455999970436096
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,4,2,128,1,float16,float16,0,0.01563999950885773
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,4,4,128,1,float16,float16,0,0.01531040072441101
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,4,1,128,1,float16,float16,0,0.015196800231933594
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,4,2,128,1,float16,float16,0,0.015294399857521058
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,4,4,128,1,float16,float16,0,0.014920000731945039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,4,1,128,1,float16,float16,0,0.014934399724006652
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,4,2,128,1,float16,float16,0,0.015020799636840821
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,4,4,128,1,float16,float16,0,0.014449599385261535
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,4,1,128,1,float16,float16,0,0.014660799503326416
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,4,2,128,1,float16,float16,0,0.014798399806022645
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,4,4,128,1,float16,float16,0,0.01441120058298111
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,4,1,128,1,float16,float16,0,0.014772799611091614
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,4,2,128,1,float16,float16,0,0.014787200093269347
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,4,1,128,1,float16,float16,0,0.03349280059337616
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,4,2,128,1,float16,float16,0,0.018268799781799315
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,4,2,128,1,float16,float16,0,0.049932798743247984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,4,2,128,1,float16,float16,0,0.023531199991703035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,4,4,128,1,float16,float16,0,0.040361601114273074
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,4,2,128,1,float16,float16,0,0.027622398734092713
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,4,4,128,1,float16,float16,0,0.017948800325393678
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,4,4,128,1,float16,float16,0,0.026516801118850707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,4,1,128,1,float16,float16,0,0.017078399658203125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,4,2,128,1,float16,float16,0,0.019284799695014954
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,4,1,128,1,float16,float16,0,0.061908799409866336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,4,4,128,1,float16,float16,0,0.018535999953746794
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,4,1,128,1,float16,float16,0,0.01592320054769516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,4,2,128,1,float16,float16,0,0.015982399880886077
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,4,1,128,1,float16,float16,0,0.02227199971675873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,4,1,128,1,float16,float16,0,0.01526239961385727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,4,2,128,1,float16,float16,0,0.014867199957370758
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,4,4,128,1,float16,float16,0,0.01480800062417984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,4,2,128,1,float16,float16,0,0.04304159879684448
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,4,1,128,1,float16,float16,0,0.01475519984960556
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,4,2,128,1,float16,float16,0,0.01478399932384491
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,4,1,128,1,float16,float16,0,0.014718399941921234
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,4,2,128,1,float16,float16,0,0.014353600144386292
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,4,4,128,1,float16,float16,0,0.014326399564743042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,4,1,128,1,float16,float16,0,0.014900800585746766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,4,2,128,1,float16,float16,0,0.014315199851989747
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,4,4,128,1,float16,float16,0,0.014206400513648987
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,4,1,128,1,float16,float16,0,0.014681600034236908
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,4,2,128,1,float16,float16,0,0.014627200365066529
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,4,1,128,1,float16,float16,0,0.028417599201202393
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,4,2,128,1,float16,float16,0,0.036457601189613345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,4,4,128,1,float16,float16,0,0.03455039858818054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,4,2,128,1,float16,float16,0,0.02595840096473694
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,4,4,128,1,float16,float16,0,0.024670399725437164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,4,1,128,1,float16,float16,0,0.016289600729942323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,4,2,128,1,float16,float16,0,0.018292799592018127
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,4,1,128,1,float16,float16,0,0.01541759967803955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,4,4,128,1,float16,float16,0,0.014838400483131408
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,4,1,128,1,float16,float16,0,0.014505599439144135
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,4,2,128,1,float16,float16,0,0.014763200283050537
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,4,4,128,1,float16,float16,0,0.014603200554847717
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,4,1,128,1,float16,float16,0,0.01446239948272705
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,4,2,128,1,float16,float16,0,0.014340800046920777
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,4,4,128,1,float16,float16,0,0.014345599710941315
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,4,1,128,1,float16,float16,0,0.014481599628925323
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,4,2,128,1,float16,float16,0,0.01432960033416748
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,4,4,128,1,float16,float16,0,0.01422560065984726
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,4,1,128,1,float16,float16,0,0.014273600280284881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,4,2,128,1,float16,float16,0,0.01419679969549179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,4,4,128,1,float16,float16,0,0.014183999598026275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,4,1,128,1,float16,float16,0,0.014318400621414184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,4,2,128,1,float16,float16,0,0.014153599739074707
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,4,1,128,1,float16,float16,0,0.02609120011329651
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,4,2,128,1,float16,float16,0,0.033792001008987424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,4,4,128,1,float16,float16,0,0.03212479948997497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,4,1,128,1,float16,float16,0,0.021324799954891206
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,4,1,128,1,float16,float16,0,0.018606400489807128
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,4,2,128,1,float16,float16,0,0.02449280023574829
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,4,4,128,1,float16,float16,0,0.023478400707244874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,4,1,128,1,float16,float16,0,0.015688000619411467
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,4,2,128,1,float16,float16,0,0.017577600479125977
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,4,4,128,1,float16,float16,0,0.01685280054807663
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,4,1,128,1,float16,float16,0,0.014865599572658539
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,4,2,128,1,float16,float16,0,0.015011200308799743
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,4,4,128,1,float16,float16,0,0.015441599488258361
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,4,4,128,1,float16,float16,0,0.014480000734329224
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,4,1,128,1,float16,float16,0,0.014235199987888336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,4,2,128,1,float16,float16,0,0.014209599792957306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,4,4,128,1,float16,float16,0,0.014508800208568573
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,4,4,128,1,float16,float16,0,0.014209599792957306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,4,1,128,1,float16,float16,0,0.014102399349212646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,4,2,128,1,float16,float16,0,0.014188799262046813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,4,4,128,1,float16,float16,0,0.014227199554443359
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,4,2,128,1,float16,float16,0,0.013940800726413728
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,4,4,128,1,float16,float16,0,0.013935999572277069
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,4,1,128,1,float16,float16,0,0.01388159990310669
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,4,2,128,1,float16,float16,0,0.013979199528694152
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,4,4,128,1,float16,float16,0,0.01404000073671341
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,4,1,128,1,float16,float16,0,0.013676799833774567
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,4,1,128,1,float16,float16,0,0.020156799256801604
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,4,2,128,1,float16,float16,0,0.013896000385284425
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,4,4,128,1,float16,float16,0,0.017716799676418305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,4,2,128,1,float16,float16,0,0.015430399775505066
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,2,2,128,1,float16,float16,0,1.7480976104736328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,2,1,128,1,float16,float16,0,1.676705551147461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,2,2,128,1,float16,float16,0,1.1869343757629394
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,2,1,128,1,float16,float16,0,2.996835136413574
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,2,2,128,1,float16,float16,0,3.0954143524169924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,4,1,128,1,float16,float16,0,0.014030399918556213
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,2,1,128,1,float16,float16,0,1.1819503784179688
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,2,2,128,1,float16,float16,0,1.0833680152893066
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,2,2,128,1,float16,float16,0,0.7672128200531005
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,2,1,128,1,float16,float16,0,1.1004704475402831
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,2,2,128,1,float16,float16,0,1.8268159866333007
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,2,1,128,1,float16,float16,0,1.832534408569336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,2,1,128,1,float16,float16,0,0.6262879848480225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,2,1,128,1,float16,float16,0,5.685414505004883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,2,1,128,1,float16,float16,0,3.272307205200195
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,2,2,128,1,float16,float16,0,0.8496239662170411
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,2,2,128,1,float16,float16,0,1.33089599609375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,2,1,128,1,float16,float16,0,0.8426095962524414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,2,2,128,1,float16,float16,0,0.6028672218322754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,2,1,128,1,float16,float16,0,0.5045807838439942
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,2,1,128,1,float16,float16,0,2.3347152709960937
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,2,2,128,1,float16,float16,0,0.9299712181091309
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,2,1,128,1,float16,float16,0,0.9061663627624512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,2,2,128,1,float16,float16,0,0.6076255798339844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,2,1,128,1,float16,float16,0,1.556436824798584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,2,2,128,1,float16,float16,0,1.5868639945983887
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,2,1,128,1,float16,float16,0,0.3615087985992432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,2,1,128,1,float16,float16,0,0.6069600105285644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,2,2,128,1,float16,float16,0,0.5972623825073242
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,2,1,128,1,float16,float16,0,0.5627840042114258
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,2,2,128,1,float16,float16,0,0.9857935905456543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,2,1,128,1,float16,float16,0,0.9390512466430664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,2,2,128,1,float16,float16,0,0.38285279273986816
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,2,1,128,1,float16,float16,0,0.251310396194458
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,2,2,128,1,float16,float16,0,0.2554415941238403
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,2,1,128,1,float16,float16,0,0.3915152072906494
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,2,1,128,1,float16,float16,0,1.6902431488037108
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,2,1,128,1,float16,float16,0,1.3225088119506836
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,2,2,128,1,float16,float16,0,0.5452335834503174
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,2,1,128,1,float16,float16,0,0.5345488071441651
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,2,2,128,1,float16,float16,0,0.3203887939453125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,2,2,128,1,float16,float16,0,1.0204863548278809
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,2,2,128,1,float16,float16,0,0.19544639587402343
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,2,1,128,1,float16,float16,0,0.30697920322418215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,2,1,128,1,float16,float16,0,0.1835551977157593
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,2,2,128,1,float16,float16,0,0.11739519834518433
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,2,1,128,1,float16,float16,0,0.11729279756546021
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,2,1,128,1,float16,float16,0,1.8509136199951173
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,2,2,128,1,float16,float16,0,0.3609584093093872
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,2,2,128,1,float16,float16,0,0.40583038330078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,2,2,128,1,float16,float16,0,0.6308127880096436
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,2,1,128,1,float16,float16,0,0.5932015895843505
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,2,2,128,1,float16,float16,0,0.21978399753570557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,2,1,128,1,float16,float16,0,0.3380768060684204
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,2,2,128,1,float16,float16,0,0.14507999420166015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,2,1,128,1,float16,float16,0,0.14260480403900147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,2,1,128,1,float16,float16,0,1.1174192428588867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,2,2,128,1,float16,float16,0,0.09315360188484192
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,2,1,128,1,float16,float16,0,0.08949120044708252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,2,1,128,1,float16,float16,0,2.859671974182129
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,2,2,128,1,float16,float16,0,0.34632000923156736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,2,2,128,1,float16,float16,0,0.20051200389862062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,2,1,128,1,float16,float16,0,0.31612000465393064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,2,2,128,1,float16,float16,0,0.6215375900268555
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,2,1,128,1,float16,float16,0,0.5720128059387207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,2,2,128,1,float16,float16,0,0.11265599727630615
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,2,1,128,1,float16,float16,0,0.19170880317687988
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,2,1,128,1,float16,float16,0,0.1058527946472168
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,2,1,128,1,float16,float16,0,1.0772336006164551
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,2,2,128,1,float16,float16,0,0.08685920238494874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,2,1,128,1,float16,float16,0,0.07643200159072876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,2,2,128,1,float16,float16,0,0.057313597202301024
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,2,1,128,1,float16,float16,0,0.056668800115585324
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,2,1,128,1,float16,float16,0,0.9761183738708497
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,2,1,128,1,float16,float16,0,0.20612800121307373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,2,2,128,1,float16,float16,0,0.22623839378356933
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,2,2,128,1,float16,float16,0,0.1378175973892212
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,2,1,128,1,float16,float16,0,0.1309008002281189
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,2,2,128,1,float16,float16,0,0.40346560478210447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,2,2,128,1,float16,float16,0,0.08498560190200806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,2,2,128,1,float16,float16,0,0.061003202199935914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,2,1,128,1,float16,float16,0,0.08248800039291382
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,2,1,128,1,float16,float16,0,0.6789472103118896
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,2,1,128,1,float16,float16,0,0.05960639715194702
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,2,1,128,1,float16,float16,0,0.04286719858646393
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,2,2,128,1,float16,float16,0,0.043224000930786134
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,2,1,128,1,float16,float16,0,0.2043760061264038
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,2,2,128,1,float16,float16,0,0.23158879280090333
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,2,2,128,1,float16,float16,0,0.13561919927597046
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,2,1,128,1,float16,float16,0,0.20691359043121338
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,2,1,128,1,float16,float16,0,0.3735055923461914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,2,2,128,1,float16,float16,0,0.42538881301879883
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,2,1,128,1,float16,float16,0,0.12352319955825805
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,2,1,128,1,float16,float16,0,0.7014048099517822
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,2,1,128,1,float16,float16,0,0.07633600234985352
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,2,2,128,1,float16,float16,0,0.08882079720497131
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,2,2,128,1,float16,float16,0,0.06901440024375916
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,2,1,128,1,float16,float16,0,0.06613600254058838
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,2,2,128,1,float16,float16,0,0.04470399916172028
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,2,1,128,1,float16,float16,0,0.04429279863834381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,2,1,128,1,float16,float16,0,0.03288959860801697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,2,2,128,1,float16,float16,0,0.033713600039482115
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,2,2,128,1,float16,float16,0,0.17880480289459227
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,2,2,128,1,float16,float16,0,0.1051584005355835
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,2,1,128,1,float16,float16,0,0.26690399646759033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,2,2,128,1,float16,float16,0,0.3282111883163452
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,2,1,128,1,float16,float16,0,0.5042367935180664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,2,2,128,1,float16,float16,0,0.06687999963760376
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,2,1,128,1,float16,float16,0,0.09010720252990723
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,2,1,128,1,float16,float16,0,0.05581120252609253
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,2,2,128,1,float16,float16,0,0.06024479866027832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,2,2,128,1,float16,float16,0,0.03630400002002716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,2,1,128,1,float16,float16,0,0.05290560126304626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,2,1,128,1,float16,float16,0,0.035996800661087035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,2,2,128,1,float16,float16,0,0.02914400100708008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,2,2,128,1,float16,float16,0,0.026900801062583923
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,2,1,128,1,float16,float16,0,0.026969599723815917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,2,1,128,1,float16,float16,0,0.3626528024673462
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,2,1,128,1,float16,float16,0,0.1451807975769043
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,2,2,128,1,float16,float16,0,0.11904640197753906
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,2,1,128,1,float16,float16,0,0.08297439813613891
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,2,2,128,1,float16,float16,0,0.06596480011940002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,2,2,128,1,float16,float16,0,0.03864319920539856
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,2,2,128,1,float16,float16,0,0.025355198979377748
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,2,1,128,1,float16,float16,0,0.023996800184249878
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,2,2,128,1,float16,float16,0,0.024131199717521666
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,2,2,128,1,float16,float16,0,0.023654399812221526
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,2,1,128,1,float16,float16,0,0.022572800517082214
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,2,2,128,1,float16,float16,0,0.023060800135135652
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,2,1,128,1,float16,float16,0,0.02228800058364868
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,2,2,128,1,float16,float16,0,0.023030400276184082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,2,1,128,1,float16,float16,0,0.07609919905662536
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,2,2,128,1,float16,float16,0,0.0619488000869751
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,2,1,128,1,float16,float16,0,0.04347040057182312
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,2,2,128,1,float16,float16,0,0.03078559935092926
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,2,1,128,1,float16,float16,0,0.025945600867271424
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,2,2,128,1,float16,float16,0,0.02098879963159561
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,2,1,128,1,float16,float16,0,0.019548800587654114
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,2,2,128,1,float16,float16,0,0.019228799641132353
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,2,1,128,1,float16,float16,0,0.1482432007789612
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,2,1,128,1,float16,float16,0,0.018156799674034118
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,2,1,128,1,float16,float16,0,0.017763200402259826
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,2,2,128,1,float16,float16,0,0.018428799510002137
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,2,2,128,1,float16,float16,0,0.01812800019979477
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,2,1,128,1,float16,float16,0,0.01737920045852661
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,2,2,128,1,float16,float16,0,0.01799200028181076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,2,1,128,1,float16,float16,0,0.017340800166130065
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,2,2,128,1,float16,float16,0,0.017929600179195405
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,2,1,128,1,float16,float16,0,0.017291200160980225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,2,1,128,1,float16,float16,0,0.04349279999732971
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,2,2,128,1,float16,float16,0,0.033817601203918454
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,2,1,128,1,float16,float16,0,0.02255840003490448
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,2,1,128,1,float16,float16,0,0.028788799047470094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,2,2,128,1,float16,float16,0,0.020452800393104553
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,2,1,128,1,float16,float16,0,0.01770080029964447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,2,2,128,1,float16,float16,0,0.01666879951953888
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,2,1,128,1,float16,float16,0,0.016310399770736693
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,2,2,128,1,float16,float16,0,0.015108799934387207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,2,1,128,1,float16,float16,0,0.015433600544929505
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,2,1,128,1,float16,float16,0,0.014937600493431092
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,2,1,128,1,float16,float16,0,0.05027840137481689
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,2,2,128,1,float16,float16,0,0.014627200365066529
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,2,1,128,1,float16,float16,0,0.03353599905967712
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,2,1,128,1,float16,float16,0,0.01478240042924881
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,2,1,128,1,float16,float16,0,0.014843200147151948
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,2,2,128,1,float16,float16,0,0.014560000598430633
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,2,2,128,1,float16,float16,0,0.01448799967765808
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,2,1,128,1,float16,float16,0,0.014715200662612915
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,2,1,128,1,float16,float16,0,0.02290399968624115
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,2,2,128,1,float16,float16,0,0.027348798513412476
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,2,1,128,1,float16,float16,0,0.020766399800777435
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,2,2,128,1,float16,float16,0,0.018798400461673737
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,2,1,128,1,float16,float16,0,0.016673600673675536
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,2,2,128,1,float16,float16,0,0.015833599865436553
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,2,1,128,1,float16,float16,0,0.015659199655056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,2,2,128,1,float16,float16,0,0.01462399959564209
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,2,1,128,1,float16,float16,0,0.014801600575447082
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,2,2,128,1,float16,float16,0,0.014412799477577209
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,2,1,128,1,float16,float16,0,0.014550399780273438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,2,2,128,1,float16,float16,0,0.014294399321079254
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,2,1,128,1,float16,float16,0,0.014422400295734406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,2,2,128,1,float16,float16,0,0.014259199798107147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,2,1,128,1,float16,float16,0,0.014424000680446625
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,2,2,128,1,float16,float16,0,0.014295999705791474
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,2,1,128,1,float16,float16,0,0.014144000411033631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,2,1,128,1,float16,float16,0,0.027798399329185486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,2,2,128,1,float16,float16,0,0.025707200169563293
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,2,2,128,1,float16,float16,0,0.01826400011777878
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,2,1,128,1,float16,float16,0,0.019431999325752257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,2,1,128,1,float16,float16,0,0.016147199273109435
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,2,1,128,1,float16,float16,0,0.015316799283027649
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,2,2,128,1,float16,float16,0,0.015559999644756317
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,2,1,128,1,float16,float16,0,0.014339199662208557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,2,2,128,1,float16,float16,0,0.014289599657058717
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,2,1,128,1,float16,float16,0,0.014169600605964661
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,2,2,128,1,float16,float16,0,0.014144000411033631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,2,1,128,1,float16,float16,0,0.014201599359512328
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,2,2,128,1,float16,float16,0,0.01420000046491623
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,2,2,128,1,float16,float16,0,0.014254400134086609
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,2,1,128,1,float16,float16,0,0.014256000518798828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,2,1,128,1,float16,float16,0,0.026276800036430358
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,2,2,128,1,float16,float16,0,0.024355199933052064
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,2,1,128,1,float16,float16,0,0.01852000057697296
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,2,2,128,1,float16,float16,0,0.017526400089263917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,2,1,128,1,float16,float16,0,0.01570879966020584
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,2,2,128,1,float16,float16,0,0.015012800693511963
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,2,1,128,1,float16,float16,0,0.01496960073709488
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,2,2,128,1,float16,float16,0,0.014390400052070618
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,2,1,128,1,float16,float16,0,0.014283199608325959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,2,2,128,1,float16,float16,0,0.014270399510860444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,2,1,128,1,float16,float16,0,0.013952000439167023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,2,2,128,1,float16,float16,0,0.014091199636459351
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,2,1,128,1,float16,float16,0,0.013967999815940857
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,2,2,128,1,float16,float16,0,0.014049600064754485
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,2,2,128,1,float16,float16,0,0.014902399480342865
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,2,1,128,1,float16,float16,0,0.013982400298118591
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,2,1,128,1,float16,float16,0,0.013748799264431
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,2,2,128,1,float16,float16,0,0.013836799561977387
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,2,1,128,1,float16,float16,0,0.029918399453163148
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,2,1,128,1,float16,float16,0,0.022118400037288665
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,1,1,128,1,float16,float16,0,0.7072671890258789
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,2,2,128,1,float16,float16,0,0.014545600116252898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,1,1,128,1,float16,float16,0,1.1891632080078125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,1,1,128,1,float16,float16,0,0.48404641151428224
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,1,1,128,1,float16,float16,0,1.0701631546020507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,2,1,128,1,float16,float16,0,0.014211200177669525
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,1,1,128,1,float16,float16,0,1.7487199783325196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,1,1,128,1,float16,float16,0,0.2386336088180542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,1,1,128,1,float16,float16,0,0.7842912197113037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,1,1,128,1,float16,float16,0,0.19477280378341674
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,1,1,128,1,float16,float16,0,0.583131217956543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,1,1,128,1,float16,float16,0,0.35098240375518797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,1,1,128,1,float16,float16,0,0.15106240510940552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,1,1,128,1,float16,float16,0,0.9940064430236817
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,1,1,128,1,float16,float16,0,0.37478559017181395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,1,1,128,1,float16,float16,0,0.6184720039367676
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,1,1,128,1,float16,float16,0,0.1902176022529602
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,1,1,128,1,float16,float16,0,0.11762720346450806
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,1,1,128,1,float16,float16,0,0.33787200450897215
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,1,1,128,1,float16,float16,0,0.0829967975616455
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,1,1,128,1,float16,float16,0,0.569817590713501
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,1,1,128,1,float16,float16,0,0.14497439861297606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,1,1,128,1,float16,float16,0,0.09143360257148743
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,1,1,128,1,float16,float16,0,0.22822721004486085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,1,1,128,1,float16,float16,0,0.366212797164917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,1,1,128,1,float16,float16,0,0.05658239722251892
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,1,1,128,1,float16,float16,0,0.08587999939918518
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,1,1,128,1,float16,float16,0,0.20787520408630372
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,1,1,128,1,float16,float16,0,0.05761920213699341
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,1,1,128,1,float16,float16,0,0.3486560106277466
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,1,1,128,1,float16,float16,0,0.042161598801612854
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,1,1,128,1,float16,float16,0,0.0851423978805542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,1,1,128,1,float16,float16,0,0.14621119499206542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,1,1,128,1,float16,float16,0,0.061667197942733766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,1,1,128,1,float16,float16,0,0.23081600666046143
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,1,1,128,1,float16,float16,0,0.04374400079250336
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,1,1,128,1,float16,float16,0,0.03324320018291473
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,1,1,128,1,float16,float16,0,0.43212318420410156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,1,1,128,1,float16,float16,0,0.08847839832305908
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,1,1,128,1,float16,float16,0,0.13872159719467164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,1,1,128,1,float16,float16,0,0.06878719925880432
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,1,1,128,1,float16,float16,0,0.2377824068069458
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,1,1,128,1,float16,float16,0,0.0332863986492157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,1,1,128,1,float16,float16,0,0.03178719878196716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,1,1,128,1,float16,float16,0,0.06720960140228271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,1,1,128,1,float16,float16,0,0.05968800187110901
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,1,1,128,1,float16,float16,0,0.18066879510879516
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,1,1,128,1,float16,float16,0,0.2176111936569214
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,1,1,128,1,float16,float16,0,0.028910401463508605
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,1,1,128,1,float16,float16,0,0.026280000805854797
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,1,1,128,1,float16,float16,0,0.02703840136528015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,1,1,128,1,float16,float16,0,0.06689919829368592
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,1,1,128,1,float16,float16,0,0.03871519863605499
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,1,1,128,1,float16,float16,0,0.026505601406097413
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,1,1,128,1,float16,float16,0,0.02462719976902008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,1,1,128,1,float16,float16,0,0.023151999711990355
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,1,1,128,1,float16,float16,0,0.02367520034313202
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,1,1,128,1,float16,float16,0,0.02324800044298172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,1,1,128,1,float16,float16,0,0.02290399968624115
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,1,1,128,1,float16,float16,0,0.031044799089431762
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,1,1,128,1,float16,float16,0,0.022198399901390074
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,1,1,128,1,float16,float16,0,0.01998399943113327
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,1,1,128,1,float16,float16,0,0.01858399957418442
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,1,1,128,1,float16,float16,0,0.018251200020313264
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,1,1,128,1,float16,float16,0,0.018083199858665466
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,1,1,128,1,float16,float16,0,0.01783519983291626
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,1,1,128,1,float16,float16,0,0.017880000174045563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,1,1,128,1,float16,float16,0,0.021953600645065307
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,1,1,128,1,float16,float16,0,0.01727360039949417
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,1,1,128,1,float16,float16,0,0.015995199978351592
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,1,1,128,1,float16,float16,0,0.014924800395965577
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,1,1,128,1,float16,float16,0,0.014659200608730317
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,1,1,128,1,float16,float16,0,0.014593599736690522
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,1,1,128,1,float16,float16,0,0.01436000019311905
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,1,1,128,1,float16,float16,0,0.014326399564743042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,1,1,128,1,float16,float16,0,0.020304000377655028
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,1,1,128,1,float16,float16,0,0.016484799981117248
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,1,1,128,1,float16,float16,0,0.01539199948310852
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,1,1,128,1,float16,float16,0,0.12624640464782716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,1,1,128,1,float16,float16,0,0.014804799854755402
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,1,1,128,1,float16,float16,0,0.01438400000333786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,1,1,128,1,float16,float16,0,0.014366400241851807
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,1,1,128,1,float16,float16,0,0.014158399403095245
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,1,1,128,1,float16,float16,0,0.01435679942369461
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,1,1,128,1,float16,float16,0,0.01945279985666275
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,1,1,128,1,float16,float16,0,0.01441120058298111
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,1,1,128,1,float16,float16,0,0.014190399646759033
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,1,1,128,1,float16,float16,0,0.014283199608325959
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,1,1,128,1,float16,float16,0,0.014244799315929414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,1,1,128,1,float16,float16,0,0.7715904235839843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,1,1,128,1,float16,float16,0,0.014281600713729858
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,1,1,128,1,float16,float16,0,0.018593600392341612
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,1,1,128,1,float16,float16,0,0.015787200629711153
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,1,1,128,1,float16,float16,0,0.014864000678062438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,1,1,128,1,float16,float16,0,0.014219200611114502
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,1,1,128,1,float16,float16,0,0.014124800264835358
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,1,1,128,1,float16,float16,0,0.014123199880123139
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,1,1,128,1,float16,float16,0,0.013915200531482697
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,1,1,128,1,float16,float16,0,0.014137600362300873
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,1,1,128,1,float16,float16,0,0.04440479874610901
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,1,1,128,1,float16,float16,0,0.10672160387039184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,1,1,128,1,float16,float16,0,0.03612639904022217
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,1,1,128,1,float16,float16,0,0.01607840061187744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,1,1,128,1,float16,float16,0,0.015223999321460725
