framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,96,1,128,1,float16,fp8,0,77.63715209960938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,96,1,128,1,fp8,fp8,0,77.53198852539063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,96,2,128,1,float16,fp8,0,77.70040893554688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,96,2,128,1,fp8,fp8,0,78.23168334960937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,96,1,128,1,float16,float16,0,92.95401000976562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,96,4,128,1,float16,fp8,0,78.02510986328124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,96,2,128,1,float16,float16,0,94.05612182617188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,96,4,128,1,float16,float16,0,94.59857788085938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,96,4,128,1,fp8,fp8,0,78.06082763671876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,96,8,128,1,float16,fp8,0,77.8643310546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,96,8,128,1,fp8,fp8,0,78.12827758789062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,1,128,1,float16,float16,0,46.879861450195314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,1,128,1,float16,fp8,0,38.92911376953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,1,128,1,fp8,fp8,0,38.78539123535156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,2,128,1,float16,float16,0,46.03042907714844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,2,128,1,float16,fp8,0,39.49080200195313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,2,128,1,fp8,fp8,0,38.9960693359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,4,128,1,float16,float16,0,46.16138916015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,4,128,1,float16,fp8,0,38.985589599609376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,4,128,1,fp8,fp8,0,39.50242919921875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,8,128,1,float16,fp8,0,39.03471374511719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,8,128,1,fp8,fp8,0,39.143798828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,96,128,1,float16,float16,0,24.010394287109374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,8,128,1,float16,float16,0,46.72349243164062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,96,128,1,float16,fp8,0,19.610749816894533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,96,128,1,fp8,fp8,0,19.697727966308594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,1,128,1,float16,fp8,0,19.114625549316408
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,96,128,1,float16,float16,0,48.681768798828124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,1,128,1,float16,float16,0,24.065863037109374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,96,128,1,float16,fp8,0,40.194891357421874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,1,128,1,fp8,fp8,0,19.755235290527345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,96,96,128,1,fp8,fp8,0,40.495660400390626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,2,128,1,float16,float16,0,23.988543701171874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,2,128,1,float16,fp8,0,19.843772888183594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,2,128,1,fp8,fp8,0,19.31439971923828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,96,8,128,1,float16,float16,0,92.47789916992187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,4,128,1,float16,fp8,0,19.209347534179688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,4,128,1,fp8,fp8,0,19.210835266113282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,4,128,1,float16,float16,0,23.108567810058595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,8,128,1,float16,float16,0,23.426878356933592
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,96,128,1,float16,float16,0,12.547444915771484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,96,128,1,float16,fp8,0,9.823849487304688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,8,128,1,float16,fp8,0,19.12733154296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,96,128,1,fp8,fp8,0,9.973095703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,1,128,1,float16,float16,0,11.712474822998047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,1,128,1,float16,fp8,0,9.812423706054688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,96,8,128,1,fp8,fp8,0,19.344346618652345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,1,128,1,fp8,fp8,0,9.801023864746094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,2,128,1,float16,fp8,0,9.580745697021484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,2,128,1,fp8,fp8,0,9.667407989501953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,2,128,1,float16,float16,0,11.724905395507813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,4,128,1,float16,float16,0,12.110553741455078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,4,128,1,float16,fp8,0,9.741226959228516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,4,128,1,fp8,fp8,0,9.890126037597657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,8,128,1,float16,float16,0,11.873603057861327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,8,128,1,float16,fp8,0,9.744535827636719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,96,8,128,1,fp8,fp8,0,9.671011352539063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,96,1,128,1,float16,fp8,0,44.753240966796874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,96,1,128,1,fp8,fp8,0,44.506509399414064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,96,2,128,1,float16,fp8,0,45.24413146972656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,96,2,128,1,fp8,fp8,0,44.59080200195312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,96,1,128,1,float16,float16,0,52.41434326171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,96,4,128,1,float16,fp8,0,44.270379638671876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,96,2,128,1,float16,float16,0,52.501251220703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,96,4,128,1,float16,float16,0,53.72359619140625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,96,128,1,float16,fp8,0,22.9205078125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,96,128,1,fp8,fp8,0,23.017715454101562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,96,128,1,float16,float16,0,27.678814697265626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,1,128,1,float16,float16,0,26.6771240234375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,96,4,128,1,fp8,fp8,0,44.97789611816406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,96,8,128,1,float16,fp8,0,45.37166748046875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,96,8,128,1,fp8,fp8,0,44.85649108886719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,1,128,1,float16,fp8,0,22.087591552734374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,96,8,128,1,float16,float16,0,53.205224609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,1,128,1,fp8,fp8,0,22.109779357910156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,2,128,1,float16,float16,0,26.096878051757812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,2,128,1,float16,fp8,0,22.258837890625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,2,128,1,fp8,fp8,0,22.539450073242186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,4,128,1,float16,fp8,0,22.205934143066408
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,4,128,1,float16,float16,0,27.233941650390626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,4,128,1,fp8,fp8,0,22.195281982421875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,8,128,1,float16,fp8,0,22.3254150390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,96,128,1,float16,fp8,0,11.552056121826173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,96,128,1,float16,float16,0,14.374478149414063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,8,128,1,float16,float16,0,26.83775939941406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,96,8,128,1,fp8,fp8,0,22.326077270507813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,96,128,1,fp8,fp8,0,11.667005157470703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,1,128,1,float16,fp8,0,10.943990325927734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,1,128,1,fp8,fp8,0,11.008523559570312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,1,128,1,float16,float16,0,13.22313232421875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,2,128,1,float16,fp8,0,11.001608276367188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,2,128,1,float16,float16,0,13.489413452148437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,2,128,1,fp8,fp8,0,10.897113800048828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,4,128,1,float16,fp8,0,11.02679672241211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,4,128,1,float16,float16,0,13.273526000976563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,4,128,1,fp8,fp8,0,10.995887756347656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,96,128,1,float16,fp8,0,5.651359939575196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,96,128,1,float16,float16,0,7.1080322265625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,8,128,1,float16,fp8,0,11.202667236328125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,8,128,1,float16,float16,0,13.261701965332032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,96,8,128,1,fp8,fp8,0,11.064217376708985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,1,128,1,float16,fp8,0,5.503612899780274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,1,128,1,float16,float16,0,6.543262481689453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,1,128,1,fp8,fp8,0,5.500099182128906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,96,128,1,fp8,fp8,0,5.67022705078125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,2,128,1,float16,fp8,0,5.596380615234375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,2,128,1,float16,float16,0,6.628321838378906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,2,128,1,fp8,fp8,0,5.559368133544922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,4,128,1,float16,fp8,0,5.509572982788086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,4,128,1,float16,float16,0,6.723561859130859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,4,128,1,fp8,fp8,0,5.564255905151367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,8,128,1,float16,fp8,0,5.683574295043945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,8,128,1,float16,float16,0,6.370880126953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,96,8,128,1,fp8,fp8,0,5.516463851928711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,96,1,128,1,float16,fp8,0,30.666165161132813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,96,1,128,1,fp8,fp8,0,31.187704467773436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,96,2,128,1,float16,fp8,0,31.08548583984375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,96,2,128,1,fp8,fp8,0,31.740341186523438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,96,4,128,1,float16,fp8,0,31.64080810546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,96,1,128,1,float16,float16,0,37.18787841796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,96,2,128,1,float16,float16,0,36.566357421875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,96,4,128,1,float16,float16,0,37.525106811523436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,96,128,1,float16,fp8,0,16.643157958984375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,96,128,1,fp8,fp8,0,16.071189880371094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,96,128,1,float16,float16,0,19.817585754394532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,1,128,1,float16,float16,0,18.20170440673828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,96,4,128,1,fp8,fp8,0,30.936648559570312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,96,8,128,1,float16,fp8,0,31.43720703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,96,8,128,1,fp8,fp8,0,31.751544189453124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,1,128,1,float16,fp8,0,15.499386596679688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,96,8,128,1,float16,float16,0,37.14667663574219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,1,128,1,fp8,fp8,0,15.325625610351562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,2,128,1,float16,float16,0,18.845738220214844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,2,128,1,float16,fp8,0,15.716954040527344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,2,128,1,fp8,fp8,0,15.800396728515626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,4,128,1,float16,fp8,0,15.89002685546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,4,128,1,float16,float16,0,18.68116455078125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,96,128,1,float16,float16,0,9.82158203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,4,128,1,fp8,fp8,0,15.922329711914063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,8,128,1,float16,fp8,0,15.685415649414063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,96,128,1,float16,fp8,0,8.03717269897461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,8,128,1,float16,float16,0,18.618986511230467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,96,8,128,1,fp8,fp8,0,15.56881103515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,96,128,1,fp8,fp8,0,8.149352264404296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,1,128,1,float16,float16,0,9.201165008544923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,1,128,1,float16,fp8,0,7.6784515380859375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,1,128,1,fp8,fp8,0,7.8664909362792965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,2,128,1,float16,fp8,0,7.66195068359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,2,128,1,float16,float16,0,9.368297576904297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,2,128,1,fp8,fp8,0,7.680693054199219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,4,128,1,float16,fp8,0,7.861459350585937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,4,128,1,float16,float16,0,9.4345458984375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,96,128,1,float16,float16,0,4.697323226928711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,4,128,1,fp8,fp8,0,7.66882095336914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,96,128,1,float16,fp8,0,3.9908687591552736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,8,128,1,float16,fp8,0,7.793316650390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,8,128,1,fp8,fp8,0,7.743470764160156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,96,8,128,1,float16,float16,0,9.468328094482422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,96,128,1,fp8,fp8,0,4.135337448120117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,1,128,1,float16,fp8,0,3.883011245727539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,1,128,1,fp8,fp8,0,3.795912170410156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,1,128,1,float16,float16,0,4.669283294677735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,2,128,1,float16,fp8,0,3.782787322998047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,2,128,1,float16,float16,0,4.599035263061523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,2,128,1,fp8,fp8,0,3.8594894409179688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,4,128,1,float16,float16,0,4.6370800018310545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,4,128,1,float16,fp8,0,3.9204273223876953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,4,128,1,fp8,fp8,0,3.794270324707031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,8,128,1,float16,fp8,0,3.9839103698730467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,8,128,1,float16,float16,0,4.580350494384765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,96,8,128,1,fp8,fp8,0,3.9233070373535157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,96,1,128,1,float16,fp8,0,40.37744140625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,96,1,128,1,fp8,fp8,0,40.62388610839844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,96,2,128,1,float16,fp8,0,40.50732421875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,96,2,128,1,fp8,fp8,0,41.07296447753906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,96,4,128,1,float16,fp8,0,41.03659362792969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,96,1,128,1,float16,float16,0,47.687567138671874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,96,2,128,1,float16,float16,0,48.91089782714844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,96,4,128,1,float16,float16,0,48.35564270019531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,96,128,1,float16,fp8,0,21.35791473388672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,96,128,1,float16,float16,0,26.32703552246094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,96,128,1,fp8,fp8,0,21.46129455566406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,1,128,1,float16,float16,0,24.06597900390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,96,4,128,1,fp8,fp8,0,40.609283447265625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,96,8,128,1,float16,fp8,0,40.77248840332031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,96,8,128,1,fp8,fp8,0,41.367434692382815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,1,128,1,float16,fp8,0,20.317796325683595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,96,8,128,1,float16,float16,0,48.61622924804688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,1,128,1,fp8,fp8,0,20.24060974121094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,2,128,1,float16,fp8,0,20.079058837890624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,2,128,1,float16,float16,0,24.86344757080078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,2,128,1,fp8,fp8,0,20.26822509765625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,4,128,1,float16,fp8,0,20.391465759277345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,4,128,1,float16,float16,0,24.030271911621092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,4,128,1,fp8,fp8,0,20.27013244628906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,96,128,1,float16,float16,0,12.891172790527344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,96,128,1,float16,fp8,0,10.67381591796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,8,128,1,float16,fp8,0,20.844004821777343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,8,128,1,float16,float16,0,24.251622009277344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,96,8,128,1,fp8,fp8,0,20.557550048828126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,96,128,1,fp8,fp8,0,10.855020904541016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,1,128,1,float16,float16,0,11.818341064453126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,1,128,1,float16,fp8,0,10.046046447753906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,1,128,1,fp8,fp8,0,10.171218872070312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,2,128,1,float16,fp8,0,10.133975982666016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,2,128,1,float16,float16,0,12.168972778320313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,2,128,1,fp8,fp8,0,10.050794982910157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,4,128,1,float16,float16,0,12.32642059326172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,4,128,1,float16,fp8,0,9.920681762695313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,4,128,1,fp8,fp8,0,10.209584045410157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,96,128,1,float16,float16,0,6.4358642578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,8,128,1,float16,fp8,0,10.29126434326172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,96,128,1,float16,fp8,0,5.200815963745117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,8,128,1,float16,float16,0,12.569569396972657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,96,8,128,1,fp8,fp8,0,10.088878631591797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,1,128,1,float16,float16,0,5.9818462371826175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,1,128,1,fp8,fp8,0,5.086702346801758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,96,128,1,fp8,fp8,0,5.322875213623047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,2,128,1,fp8,fp8,0,5.142380905151367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,2,128,1,float16,float16,0,6.237323379516601
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,4,128,1,float16,fp8,0,5.120089721679688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,4,128,1,float16,float16,0,5.998241424560547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,4,128,1,fp8,fp8,0,5.024643325805664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,2,128,1,float16,fp8,0,4.9675647735595705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,1,128,1,float16,fp8,0,4.977008056640625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,96,128,1,float16,fp8,0,2.796793556213379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,96,128,1,float16,float16,0,3.1684640884399413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,8,128,1,float16,fp8,0,5.078339385986328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,8,128,1,fp8,fp8,0,5.1831520080566404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,1,128,1,float16,float16,0,2.7342464447021486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,1,128,1,float16,fp8,0,2.4461952209472657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,1,128,1,fp8,fp8,0,2.474715232849121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,2,128,1,float16,fp8,0,2.5054767608642576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,2,128,1,float16,float16,0,2.920568084716797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,2,128,1,fp8,fp8,0,2.546641540527344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,4,128,1,float16,fp8,0,2.481119918823242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,4,128,1,float16,float16,0,2.8669551849365233
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,96,128,1,fp8,fp8,0,2.592075157165527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,4,128,1,fp8,fp8,0,2.494028854370117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,8,128,1,float16,float16,0,2.984395217895508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,8,128,1,fp8,fp8,0,2.502524757385254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,96,8,128,1,float16,float16,0,5.860095977783203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,96,8,128,1,float16,fp8,0,2.614756774902344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,96,1,128,1,float16,fp8,0,23.07920684814453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,96,1,128,1,fp8,fp8,0,23.10993194580078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,96,2,128,1,fp8,fp8,0,23.56680908203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,96,2,128,1,float16,fp8,0,23.793777465820312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,96,1,128,1,float16,float16,0,27.111239624023437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,96,2,128,1,float16,float16,0,27.745687866210936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,96,4,128,1,float16,fp8,0,23.001820373535157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,96,4,128,1,float16,float16,0,27.64881591796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,96,128,1,float16,fp8,0,12.34974365234375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,96,128,1,fp8,fp8,0,12.735116577148437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,96,128,1,float16,float16,0,15.207136535644532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,1,128,1,float16,float16,0,14.7114501953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,96,4,128,1,fp8,fp8,0,23.534283447265626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,96,8,128,1,float16,fp8,0,23.758090209960937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,96,8,128,1,fp8,fp8,0,23.897918701171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,96,8,128,1,float16,float16,0,28.759201049804688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,1,128,1,float16,fp8,0,11.678543853759766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,1,128,1,fp8,fp8,0,11.531924438476562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,2,128,1,float16,float16,0,13.612141418457032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,2,128,1,float16,fp8,0,11.576214599609376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,2,128,1,fp8,fp8,0,11.572752380371094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,4,128,1,float16,fp8,0,11.981070709228515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,4,128,1,float16,float16,0,14.309315490722657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,8,128,1,float16,fp8,0,11.543238067626953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,96,128,1,float16,fp8,0,6.2366081237792965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,96,128,1,float16,float16,0,7.389313507080078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,8,128,1,float16,float16,0,14.200054931640626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,4,128,1,fp8,fp8,0,11.578717041015626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,96,8,128,1,fp8,fp8,0,11.708889770507813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,96,128,1,fp8,fp8,0,6.290249633789062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,1,128,1,float16,float16,0,6.962108612060547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,1,128,1,float16,fp8,0,5.692580795288086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,1,128,1,fp8,fp8,0,5.786339187622071
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,2,128,1,float16,float16,0,6.892588806152344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,2,128,1,fp8,fp8,0,5.737459182739258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,4,128,1,float16,fp8,0,5.927395248413086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,4,128,1,float16,float16,0,7.023323059082031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,4,128,1,fp8,fp8,0,5.843683242797852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,96,128,1,float16,float16,0,3.641561508178711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,2,128,1,float16,fp8,0,5.868801498413086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,8,128,1,float16,fp8,0,5.8764606475830075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,96,128,1,float16,fp8,0,3.1091552734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,8,128,1,float16,float16,0,7.301369476318359
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,96,128,1,fp8,fp8,0,3.1469423294067385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,96,8,128,1,fp8,fp8,0,6.0304607391357425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,1,128,1,fp8,fp8,0,2.8312944412231444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,1,128,1,float16,fp8,0,2.9765695571899413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,1,128,1,float16,float16,0,3.359640121459961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,2,128,1,float16,fp8,0,2.9385520935058596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,2,128,1,float16,float16,0,3.3972705841064452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,2,128,1,fp8,fp8,0,2.9979904174804686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,4,128,1,float16,fp8,0,3.009611129760742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,4,128,1,float16,float16,0,3.3268352508544923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,4,128,1,fp8,fp8,0,2.86397762298584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,96,128,1,float16,float16,0,1.9558687210083008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,8,128,1,float16,fp8,0,3.01834716796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,96,128,1,fp8,fp8,0,1.587007999420166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,1,128,1,float16,float16,0,1.656870460510254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,1,128,1,float16,fp8,0,1.4399567604064942
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,1,128,1,fp8,fp8,0,1.5124688148498535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,2,128,1,float16,float16,0,1.583345603942871
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,2,128,1,float16,fp8,0,1.5450896263122558
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,2,128,1,fp8,fp8,0,1.4325856208801269
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,4,128,1,float16,float16,0,1.5422224044799804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,4,128,1,float16,fp8,0,1.546617603302002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,4,128,1,fp8,fp8,0,1.4565360069274902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,8,128,1,float16,float16,0,3.205718231201172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,96,8,128,1,fp8,fp8,0,2.946656036376953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,8,128,1,float16,float16,0,1.5197440147399903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,8,128,1,float16,fp8,0,1.5233807563781738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,8,128,1,fp8,fp8,0,1.4761856079101563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,96,96,128,1,float16,fp8,0,1.5377280235290527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,96,1,128,1,float16,fp8,0,21.5906494140625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,96,1,128,1,fp8,fp8,0,21.6626220703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,96,2,128,1,float16,fp8,0,21.679537963867187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,96,2,128,1,fp8,fp8,0,22.228565979003907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,96,1,128,1,float16,float16,0,25.515733337402345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,96,2,128,1,float16,float16,0,25.878170776367188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,96,4,128,1,float16,float16,0,25.215765380859374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,96,4,128,1,float16,fp8,0,21.68848419189453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,96,128,1,float16,fp8,0,11.799366760253907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,96,128,1,float16,float16,0,14.457830810546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,96,128,1,fp8,fp8,0,12.060697937011719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,1,128,1,float16,float16,0,13.058407592773438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,96,4,128,1,fp8,fp8,0,22.381857299804686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,96,8,128,1,float16,fp8,0,22.25122985839844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,96,8,128,1,fp8,fp8,0,22.228326416015626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,96,8,128,1,float16,float16,0,27.32787170410156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,1,128,1,float16,fp8,0,10.715184020996094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,1,128,1,fp8,fp8,0,10.73639373779297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,2,128,1,float16,fp8,0,11.067030334472657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,2,128,1,float16,float16,0,12.758190155029297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,2,128,1,fp8,fp8,0,10.878604888916016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,4,128,1,float16,fp8,0,11.134897613525391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,4,128,1,float16,float16,0,12.959352111816406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,4,128,1,fp8,fp8,0,10.921737670898438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,96,128,1,float16,float16,0,6.956050872802734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,96,128,1,float16,fp8,0,5.887276840209961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,8,128,1,float16,fp8,0,10.902458953857423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,8,128,1,float16,float16,0,13.290631103515626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,96,128,1,fp8,fp8,0,5.986867141723633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,1,128,1,float16,float16,0,6.293915176391602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,96,8,128,1,fp8,fp8,0,10.975902557373047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,1,128,1,float16,fp8,0,5.369334411621094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,1,128,1,fp8,fp8,0,5.378774261474609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,2,128,1,float16,fp8,0,5.392406463623047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,2,128,1,float16,float16,0,6.44042739868164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,2,128,1,fp8,fp8,0,5.471038436889648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,4,128,1,float16,float16,0,6.62109603881836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,4,128,1,float16,fp8,0,5.42803840637207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,4,128,1,fp8,fp8,0,5.431243133544922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,96,128,1,float16,float16,0,3.6495296478271486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,96,128,1,float16,fp8,0,2.919211196899414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,8,128,1,float16,fp8,0,5.378817749023438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,8,128,1,float16,float16,0,6.5103759765625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,96,8,128,1,fp8,fp8,0,5.448767852783203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,1,128,1,float16,fp8,0,2.676715278625488
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,96,128,1,fp8,fp8,0,3.0426752090454103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,1,128,1,fp8,fp8,0,2.8826799392700195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,2,128,1,float16,fp8,0,2.680041694641113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,2,128,1,fp8,fp8,0,2.881395149230957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,4,128,1,float16,fp8,0,2.679254341125488
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,4,128,1,fp8,fp8,0,2.7049999237060547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,4,128,1,float16,float16,0,3.2652000427246093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,1,128,1,float16,float16,0,3.132809638977051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,8,128,1,float16,fp8,0,2.6946319580078124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,96,128,1,float16,fp8,0,1.4557295799255372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,8,128,1,float16,float16,0,2.9632543563842773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,8,128,1,fp8,fp8,0,2.7193424224853517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,96,128,1,fp8,fp8,0,1.653887939453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,96,2,128,1,float16,float16,0,2.9200191497802734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,1,128,1,float16,fp8,0,1.3399200439453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,2,128,1,float16,float16,0,1.6167343139648438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,2,128,1,fp8,fp8,0,1.4848048210144043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,4,128,1,float16,float16,0,1.4239312171936036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,4,128,1,float16,fp8,0,1.3437503814697265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,4,128,1,fp8,fp8,0,1.4640959739685058
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,8,128,1,float16,float16,0,1.4536224365234376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,96,128,1,float16,float16,0,1.6814176559448242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,8,128,1,float16,fp8,0,1.4935407638549805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,8,128,1,fp8,fp8,0,1.3478464126586913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,1,128,1,float16,float16,0,1.416312026977539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,96,128,1,float16,fp8,0,0.7721776008605957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,96,128,1,fp8,fp8,0,0.7356624126434326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,1,128,1,float16,float16,0,0.7771088123321533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,1,128,1,float16,fp8,0,0.6752943992614746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,1,128,1,fp8,fp8,0,0.6779104232788086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,2,128,1,float16,float16,0,0.7400464057922364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,2,128,1,fp8,fp8,0,0.6856448173522949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,4,128,1,float16,float16,0,0.714792013168335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,4,128,1,float16,fp8,0,0.6807295799255371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,4,128,1,fp8,fp8,0,0.6935823917388916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,8,128,1,float16,float16,0,0.716542387008667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,8,128,1,float16,fp8,0,0.6793263912200928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,8,128,1,fp8,fp8,0,0.6820256233215332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,96,128,1,float16,float16,0,0.8205167770385742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,1,128,1,fp8,fp8,0,1.3549471855163575
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,96,2,128,1,float16,fp8,0,1.3480976104736329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,96,2,128,1,float16,fp8,0,0.6752816200256347
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,96,1,128,1,float16,fp8,0,12.647154998779296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,96,1,128,1,fp8,fp8,0,12.628971099853516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,96,2,128,1,float16,fp8,0,12.767839813232422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,96,2,128,1,float16,float16,0,14.870680236816407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,96,2,128,1,fp8,fp8,0,12.78533935546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,96,1,128,1,float16,float16,0,14.798992919921876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,96,4,128,1,float16,float16,0,15.019294738769531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,96,128,1,float16,float16,0,8.624537658691406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,96,4,128,1,float16,fp8,0,12.818458557128906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,96,128,1,float16,fp8,0,7.081591796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,96,4,128,1,fp8,fp8,0,12.752609252929688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,96,8,128,1,float16,fp8,0,12.868603515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,96,8,128,1,float16,float16,0,15.196278381347657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,96,128,1,fp8,fp8,0,7.142727661132812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,1,128,1,float16,fp8,0,6.3577423095703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,96,8,128,1,fp8,fp8,0,12.703998565673828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,1,128,1,float16,float16,0,7.467372894287109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,1,128,1,fp8,fp8,0,6.309273529052734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,2,128,1,float16,float16,0,7.918084716796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,2,128,1,float16,fp8,0,6.379369735717773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,2,128,1,fp8,fp8,0,6.389811325073242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,4,128,1,float16,float16,0,7.5157218933105465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,4,128,1,float16,fp8,0,6.396521759033203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,4,128,1,fp8,fp8,0,6.332651138305664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,8,128,1,float16,fp8,0,6.400759887695313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,96,128,1,float16,fp8,0,3.5245681762695313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,8,128,1,float16,float16,0,8.31571044921875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,96,8,128,1,fp8,fp8,0,6.428905487060547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,96,128,1,fp8,fp8,0,4.009280014038086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,1,128,1,float16,float16,0,3.451465606689453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,1,128,1,float16,fp8,0,3.162265586853027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,1,128,1,fp8,fp8,0,3.1540847778320313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,2,128,1,float16,float16,0,3.603144073486328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,2,128,1,fp8,fp8,0,3.143012809753418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,2,128,1,float16,fp8,0,3.3148929595947267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,4,128,1,float16,float16,0,3.7348270416259766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,4,128,1,fp8,fp8,0,3.170416069030762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,8,128,1,float16,float16,0,3.571223831176758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,8,128,1,float16,fp8,0,3.291331100463867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,96,128,1,float16,float16,0,4.513857650756836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,96,128,1,float16,fp8,0,1.8683296203613282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,96,128,1,fp8,fp8,0,1.768123245239258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,8,128,1,fp8,fp8,0,3.1790128707885743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,1,128,1,float16,fp8,0,1.5857952117919922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,1,128,1,float16,float16,0,1.8189647674560547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,1,128,1,fp8,fp8,0,1.585923194885254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,2,128,1,float16,float16,0,1.7100671768188476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,2,128,1,float16,fp8,0,1.5922112464904785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,96,4,128,1,float16,fp8,0,3.5224945068359377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,2,128,1,fp8,fp8,0,1.5896688461303712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,4,128,1,float16,float16,0,1.6654464721679687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,4,128,1,float16,fp8,0,1.6901119232177735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,4,128,1,fp8,fp8,0,1.5960960388183594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,96,128,1,float16,float16,0,2.044887924194336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,96,128,1,float16,float16,0,0.9912704467773438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,8,128,1,float16,float16,0,1.8014015197753905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,8,128,1,fp8,fp8,0,1.5869327545166017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,96,128,1,float16,fp8,0,0.951046371459961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,1,128,1,float16,float16,0,0.8402480125427246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,96,128,1,fp8,fp8,0,0.9080207824707032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,1,128,1,float16,fp8,0,0.7949999809265137
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,1,128,1,fp8,fp8,0,0.8589280128479004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,2,128,1,float16,fp8,0,0.7947743892669678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,2,128,1,float16,float16,0,0.9750255584716797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,4,128,1,float16,fp8,0,0.8098784446716308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,4,128,1,fp8,fp8,0,0.798795223236084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,8,128,1,float16,float16,0,0.8574975967407227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,96,128,1,float16,float16,0,0.5051712036132813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,8,128,1,float16,fp8,0,0.7997407913208008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,8,128,1,fp8,fp8,0,0.8717904090881348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,96,128,1,fp8,fp8,0,0.4594543933868408
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,1,128,1,float16,float16,0,0.45184640884399413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,1,128,1,float16,fp8,0,0.4040080070495605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,2,128,1,float16,float16,0,0.44142398834228513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,2,128,1,fp8,fp8,0,0.4051951885223389
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,4,128,1,float16,float16,0,0.4376495838165283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,96,8,128,1,float16,fp8,0,1.6020896911621094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,4,128,1,fp8,fp8,0,0.4135744094848633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,8,128,1,float16,float16,0,0.43025760650634765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,8,128,1,float16,fp8,0,0.4120336055755615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,2,128,1,fp8,fp8,0,0.8225584030151367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,96,4,128,1,float16,float16,0,0.8811856269836426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,96,128,1,float16,fp8,0,0.44916958808898927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,1,128,1,fp8,fp8,0,0.42527360916137696
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,2,128,1,float16,fp8,0,0.42452001571655273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,4,128,1,float16,fp8,0,0.40604801177978517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,96,8,128,1,fp8,fp8,0,0.4104127883911133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,96,1,128,1,float16,fp8,0,12.34079360961914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,96,1,128,1,fp8,fp8,0,12.367124938964844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,96,1,128,1,float16,float16,0,14.217469787597656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,96,2,128,1,float16,fp8,0,12.396590423583984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,96,2,128,1,fp8,fp8,0,12.372962951660156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,96,2,128,1,float16,float16,0,14.304478454589844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,96,4,128,1,float16,fp8,0,12.339723205566406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,96,4,128,1,float16,float16,0,14.499613952636718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,96,128,1,float16,fp8,0,7.185975646972656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,96,128,1,float16,float16,0,8.461557006835937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,96,128,1,fp8,fp8,0,7.125804901123047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,1,128,1,float16,float16,0,7.134808349609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,96,4,128,1,fp8,fp8,0,12.351048278808594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,96,8,128,1,float16,fp8,0,12.434910583496094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,96,8,128,1,fp8,fp8,0,12.537493133544922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,96,8,128,1,float16,float16,0,14.495538330078125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,1,128,1,float16,fp8,0,6.190132904052734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,1,128,1,fp8,fp8,0,6.219345474243164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,2,128,1,float16,fp8,0,6.1701904296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,2,128,1,float16,float16,0,7.183502197265625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,2,128,1,fp8,fp8,0,6.300742340087891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,4,128,1,float16,fp8,0,6.315251159667969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,4,128,1,float16,float16,0,7.363246154785156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,4,128,1,fp8,fp8,0,6.560214233398438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,96,128,1,float16,float16,0,4.625523376464844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,8,128,1,float16,fp8,0,6.218048095703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,96,128,1,float16,fp8,0,3.5560688018798827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,8,128,1,float16,float16,0,7.585655975341797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,96,128,1,fp8,fp8,0,3.7024559020996093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,96,8,128,1,fp8,fp8,0,6.302835083007812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,1,128,1,float16,float16,0,3.9151535034179688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,1,128,1,float16,fp8,0,3.115015983581543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,1,128,1,fp8,fp8,0,3.453432083129883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,2,128,1,float16,fp8,0,3.101628875732422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,2,128,1,fp8,fp8,0,3.3929039001464845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,4,128,1,float16,fp8,0,3.124662399291992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,4,128,1,float16,float16,0,3.648307037353516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,4,128,1,fp8,fp8,0,3.3849647521972654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,8,128,1,float16,float16,0,3.6057998657226564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,96,128,1,float16,float16,0,2.0344928741455077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,96,128,1,float16,fp8,0,1.7938335418701172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,8,128,1,float16,fp8,0,3.1146575927734377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,8,128,1,fp8,fp8,0,3.113462448120117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,1,128,1,float16,float16,0,1.6748176574707032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,96,2,128,1,float16,float16,0,3.411532974243164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,1,128,1,float16,fp8,0,1.676905632019043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,2,128,1,float16,float16,0,1.6630159378051759
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,1,128,1,fp8,fp8,0,1.63731689453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,2,128,1,float16,fp8,0,1.5563344001770019
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,2,128,1,fp8,fp8,0,1.5527456283569336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,4,128,1,float16,float16,0,1.7948831558227538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,4,128,1,fp8,fp8,0,1.5801360130310058
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,8,128,1,fp8,fp8,0,1.5562911987304688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,8,128,1,float16,float16,0,1.8203216552734376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,96,128,1,float16,fp8,0,0.9581664085388184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,96,128,1,fp8,fp8,0,1.8970624923706054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,96,128,1,fp8,fp8,0,0.8987504005432129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,1,128,1,float16,float16,0,0.8152144432067872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,1,128,1,float16,fp8,0,0.838646411895752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,1,128,1,fp8,fp8,0,0.8119775772094726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,2,128,1,float16,float16,0,0.8139568328857422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,2,128,1,float16,fp8,0,0.7803247928619385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,2,128,1,fp8,fp8,0,0.8027791976928711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,4,128,1,float16,fp8,0,0.7946944236755371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,4,128,1,fp8,fp8,0,0.7895343780517579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,8,128,1,float16,float16,0,0.8328031539916992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,4,128,1,float16,fp8,0,1.5693936347961426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,8,128,1,float16,fp8,0,0.7946191787719726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,96,128,1,float16,float16,0,0.5094751834869384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,8,128,1,fp8,fp8,0,0.7884912014007568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,96,128,1,float16,fp8,0,0.48721919059753416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,1,128,1,float16,float16,0,0.4122191905975342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,96,128,1,fp8,fp8,0,0.46139678955078123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,1,128,1,fp8,fp8,0,0.39576001167297364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,2,128,1,float16,float16,0,0.43172321319580076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,2,128,1,float16,fp8,0,0.395582389831543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,2,128,1,fp8,fp8,0,0.4009232044219971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,4,128,1,float16,float16,0,0.41848320960998536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,4,128,1,float16,fp8,0,0.412340784072876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,4,128,1,fp8,fp8,0,0.396727991104126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,8,128,1,float16,float16,0,0.424132776260376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,8,128,1,float16,fp8,0,0.39950239658355713
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,96,128,1,float16,float16,0,0.2630511999130249
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,8,128,1,fp8,fp8,0,0.4048783779144287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,96,128,1,float16,fp8,0,0.23905279636383056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,96,128,1,fp8,fp8,0,0.2368607997894287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,1,128,1,float16,fp8,0,0.20511360168457032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,1,128,1,fp8,fp8,0,0.20614879131317138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,2,128,1,float16,float16,0,0.21555039882659913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,2,128,1,float16,fp8,0,0.20560319423675538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,2,128,1,fp8,fp8,0,0.2065119981765747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,4,128,1,float16,float16,0,0.21693599224090576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,4,128,1,float16,float16,0,0.8380144119262696
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,4,128,1,fp8,fp8,0,0.20655200481414795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,8,128,1,float16,float16,0,0.2191296100616455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,8,128,1,float16,fp8,0,0.2074671983718872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,8,128,1,fp8,fp8,0,0.208351993560791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,96,96,128,1,float16,float16,0,1.002143955230713
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,96,1,128,1,float16,fp8,0,0.39591360092163086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,96,8,128,1,float16,fp8,0,1.575265598297119
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,1,128,1,float16,float16,0,0.21298398971557617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,96,4,128,1,float16,fp8,0,0.2060960054397583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,96,1,128,1,float16,fp8,0,7.6526031494140625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,96,1,128,1,fp8,fp8,0,7.590745544433593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,96,1,128,1,float16,float16,0,8.413674926757812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,96,2,128,1,float16,fp8,0,7.642803192138672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,96,2,128,1,fp8,fp8,0,7.594825744628906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,96,2,128,1,float16,float16,0,8.457638549804688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,96,4,128,1,float16,fp8,0,7.6678306579589846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,96,128,1,float16,fp8,0,4.581647872924805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,96,128,1,float16,float16,0,5.2680622100830075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,96,4,128,1,float16,float16,0,8.689231872558594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,96,4,128,1,fp8,fp8,0,7.626911926269531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,96,128,1,fp8,fp8,0,4.539273452758789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,96,8,128,1,float16,fp8,0,7.709798431396484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,96,8,128,1,fp8,fp8,0,7.613521575927734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,1,128,1,float16,float16,0,4.118979263305664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,1,128,1,float16,fp8,0,3.8027088165283205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,1,128,1,fp8,fp8,0,3.8128849029541017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,96,8,128,1,float16,float16,0,8.70965576171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,2,128,1,float16,float16,0,4.203152084350586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,2,128,1,fp8,fp8,0,3.7940975189208985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,4,128,1,float16,float16,0,4.184902572631836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,4,128,1,float16,fp8,0,3.832912063598633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,4,128,1,fp8,fp8,0,3.815339279174805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,2,128,1,float16,fp8,0,3.8166046142578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,8,128,1,float16,fp8,0,3.907331085205078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,8,128,1,fp8,fp8,0,3.8146385192871093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,96,128,1,float16,fp8,0,2.2527584075927733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,96,128,1,float16,float16,0,2.6142784118652345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,96,128,1,fp8,fp8,0,2.266334342956543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,1,128,1,fp8,fp8,0,1.8986991882324218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,2,128,1,float16,float16,0,2.038944053649902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,2,128,1,float16,fp8,0,1.991414451599121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,96,8,128,1,float16,float16,0,4.193616104125977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,4,128,1,float16,float16,0,2.029742431640625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,4,128,1,float16,fp8,0,1.9289039611816405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,1,128,1,float16,float16,0,1.9710224151611329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,4,128,1,fp8,fp8,0,2.0113359451293946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,1,128,1,float16,fp8,0,1.9183919906616211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,8,128,1,float16,fp8,0,1.9291215896606446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,96,128,1,float16,float16,0,1.2581279754638672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,8,128,1,fp8,fp8,0,2.0221424102783203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,96,128,1,float16,fp8,0,1.14192476272583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,96,128,1,fp8,fp8,0,1.1436256408691405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,1,128,1,float16,float16,0,0.9922143936157226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,1,128,1,float16,fp8,0,0.9611519813537598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,2,128,1,fp8,fp8,0,1.9109712600708009
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,1,128,1,fp8,fp8,0,0.9665583610534668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,2,128,1,float16,float16,0,1.0012271881103516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,2,128,1,float16,fp8,0,0.9607855796813964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,2,128,1,fp8,fp8,0,0.9606672286987304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,4,128,1,float16,float16,0,1.0017536163330079
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,4,128,1,fp8,fp8,0,0.9636863708496094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,96,128,1,float16,float16,0,0.6397647857666016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,8,128,1,float16,float16,0,1.0083984375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,8,128,1,float16,fp8,0,0.983737564086914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,8,128,1,fp8,fp8,0,0.9697360038757324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,96,8,128,1,float16,float16,0,2.026820755004883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,96,128,1,fp8,fp8,0,0.5783999919891357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,1,128,1,float16,fp8,0,0.5194496154785156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,1,128,1,fp8,fp8,0,0.48586239814758303
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,2,128,1,float16,fp8,0,0.49053277969360354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,2,128,1,fp8,fp8,0,0.48598718643188477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,4,128,1,float16,float16,0,0.5312352180480957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,4,128,1,float16,fp8,0,0.48892960548400877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,4,128,1,fp8,fp8,0,0.4921440124511719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,8,128,1,float16,float16,0,0.5071536064147949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,8,128,1,float16,fp8,0,0.49880480766296387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,96,128,1,float16,float16,0,0.3290816068649292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,96,128,1,float16,fp8,0,0.29716479778289795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,96,128,1,fp8,fp8,0,0.3008352041244507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,1,128,1,float16,fp8,0,0.24768800735473634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,1,128,1,fp8,fp8,0,0.24981439113616943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,96,4,128,1,float16,fp8,0,0.9656432151794434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,2,128,1,float16,fp8,0,0.25033600330352784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,2,128,1,fp8,fp8,0,0.24958720207214355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,4,128,1,float16,float16,0,0.2594399929046631
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,96,128,1,float16,fp8,0,0.5790016174316406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,4,128,1,float16,fp8,0,0.2508447885513306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,4,128,1,fp8,fp8,0,0.25106561183929443
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,8,128,1,float16,float16,0,0.26323039531707765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,8,128,1,float16,fp8,0,0.2518143892288208
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,8,128,1,fp8,fp8,0,0.2521503925323486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,2,128,1,float16,float16,0,0.5028175830841064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,96,128,1,float16,float16,0,0.17341760396957398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,96,128,1,float16,fp8,0,0.15633920431137086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,96,128,1,fp8,fp8,0,0.15700479745864868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,1,128,1,float16,float16,0,0.1355631947517395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,1,128,1,fp8,fp8,0,0.1295375943183899
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,2,128,1,float16,float16,0,0.13520480394363404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,2,128,1,float16,fp8,0,0.1305807948112488
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,4,128,1,float16,float16,0,0.1360592007637024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,4,128,1,float16,fp8,0,0.1318351984024048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,8,128,1,float16,float16,0,0.1385200023651123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,8,128,1,fp8,fp8,0,0.4916399955749512
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,8,128,1,float16,fp8,0,0.13184159994125366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,8,128,1,fp8,fp8,0,0.13271360397338866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,1,128,1,float16,float16,0,0.2577392101287842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,96,2,128,1,float16,float16,0,0.2552495956420898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,96,1,128,1,float16,float16,0,0.4984288215637207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,1,128,1,float16,fp8,0,0.13023359775543214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,2,128,1,fp8,fp8,0,0.1303104043006897
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,96,4,128,1,fp8,fp8,0,0.13117920160293578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,96,1,128,1,float16,fp8,0,7.940464019775391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,96,1,128,1,fp8,fp8,0,7.973844909667969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,96,1,128,1,float16,float16,0,8.578224182128906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,96,2,128,1,float16,float16,0,8.428924560546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,96,2,128,1,float16,fp8,0,7.916619110107422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,96,2,128,1,fp8,fp8,0,7.916998291015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,96,4,128,1,float16,fp8,0,8.014568328857422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,96,4,128,1,float16,float16,0,8.643177795410157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,96,128,1,float16,fp8,0,4.898561477661133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,96,128,1,float16,float16,0,5.76214714050293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,96,4,128,1,fp8,fp8,0,7.948891448974609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,96,8,128,1,float16,fp8,0,8.09288330078125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,96,128,1,fp8,fp8,0,4.9671630859375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,96,8,128,1,float16,float16,0,8.79083023071289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,96,8,128,1,fp8,fp8,0,8.104991912841797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,1,128,1,float16,float16,0,4.261017608642578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,1,128,1,fp8,fp8,0,3.9644046783447267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,2,128,1,float16,fp8,0,3.9951713562011717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,2,128,1,fp8,fp8,0,4.031727981567383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,4,128,1,float16,float16,0,4.226836776733398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,4,128,1,float16,fp8,0,4.0006561279296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,1,128,1,float16,fp8,0,4.038339233398437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,4,128,1,fp8,fp8,0,4.040544128417968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,2,128,1,float16,float16,0,4.258940887451172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,96,128,1,float16,fp8,0,2.4722848892211915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,96,128,1,float16,float16,0,2.8036575317382812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,8,128,1,float16,fp8,0,4.03948974609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,8,128,1,fp8,fp8,0,4.036700820922851
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,1,128,1,float16,float16,0,2.014344024658203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,96,128,1,fp8,fp8,0,2.474105644226074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,1,128,1,float16,fp8,0,2.009238433837891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,2,128,1,float16,float16,0,2.0436880111694338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,2,128,1,float16,fp8,0,1.9988527297973633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,2,128,1,fp8,fp8,0,2.0070335388183596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,96,8,128,1,float16,float16,0,4.286276626586914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,4,128,1,float16,float16,0,2.107671928405762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,4,128,1,float16,fp8,0,2.014614486694336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,4,128,1,fp8,fp8,0,2.0213151931762696
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,8,128,1,float16,float16,0,2.1293487548828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,8,128,1,float16,fp8,0,2.018804740905762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,8,128,1,fp8,fp8,0,2.022318458557129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,96,128,1,float16,float16,0,1.383255958557129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,1,128,1,float16,float16,0,1.0163087844848633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,96,128,1,float16,fp8,0,1.321174430847168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,96,128,1,fp8,fp8,0,1.2350303649902343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,96,1,128,1,fp8,fp8,0,1.9997119903564453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,1,128,1,float16,fp8,0,1.0662848472595214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,1,128,1,fp8,fp8,0,1.003435230255127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,2,128,1,float16,float16,0,1.0080479621887206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,2,128,1,float16,fp8,0,1.011355209350586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,2,128,1,fp8,fp8,0,1.0123200416564941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,4,128,1,float16,fp8,0,0.9989744186401367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,4,128,1,fp8,fp8,0,1.0128720283508301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,8,128,1,float16,float16,0,1.0534496307373047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,8,128,1,float16,fp8,0,1.0159024238586425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,96,128,1,float16,float16,0,0.701582384109497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,8,128,1,fp8,fp8,0,1.0116640090942384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,96,128,1,float16,fp8,0,0.6332687854766845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,1,128,1,float16,float16,0,0.5108816146850585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,1,128,1,float16,fp8,0,0.512286376953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,1,128,1,fp8,fp8,0,0.506883192062378
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,2,128,1,float16,fp8,0,0.5058879852294922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,2,128,1,fp8,fp8,0,0.5124815940856934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,4,128,1,float16,float16,0,0.5170415878295899
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,4,128,1,float16,fp8,0,0.5133743762969971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,4,128,1,fp8,fp8,0,0.5092319965362548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,8,128,1,float16,float16,0,0.523524808883667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,8,128,1,float16,fp8,0,0.5182640075683593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,96,128,1,float16,float16,0,0.359004807472229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,96,128,1,float16,fp8,0,0.32137279510498046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,96,128,1,fp8,fp8,0,0.3203423976898193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,1,128,1,float16,float16,0,0.26374719142913816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,1,128,1,float16,fp8,0,0.2606224060058594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,1,128,1,fp8,fp8,0,0.258785605430603
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,2,128,1,float16,float16,0,0.26225759983062746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,2,128,1,float16,fp8,0,0.261353588104248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,2,128,1,fp8,fp8,0,0.2622191905975342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,4,128,1,float16,float16,0,0.2649120092391968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,4,128,1,float16,fp8,0,0.2600032091140747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,4,128,1,fp8,fp8,0,0.26272480487823485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,8,128,1,float16,fp8,0,0.2623647928237915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,96,128,1,float16,float16,0,0.18933279514312745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,96,128,1,float16,fp8,0,0.16817599534988403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,96,128,1,fp8,fp8,0,0.16715999841690063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,1,128,1,float16,float16,0,0.1376479983329773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,1,128,1,float16,fp8,0,0.13384159803390502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,1,128,1,fp8,fp8,0,0.1350751996040344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,2,128,1,float16,float16,0,0.13714879751205444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,2,128,1,float16,fp8,0,0.13550879955291747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,2,128,1,fp8,fp8,0,0.134716796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,4,128,1,float16,float16,0,0.13918720483779906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,4,128,1,float16,fp8,0,0.1351248025894165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,8,128,1,float16,float16,0,0.1404736042022705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,8,128,1,float16,fp8,0,0.13625119924545287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,8,128,1,fp8,fp8,0,0.13624160289764403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,96,128,1,float16,float16,0,0.10028320550918579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,96,128,1,float16,fp8,0,0.08978239893913269
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,96,128,1,fp8,fp8,0,0.0902559995651245
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,1,128,1,float16,float16,0,0.07465919852256775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,8,128,1,float16,float16,0,0.27029440402984617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,96,8,128,1,fp8,fp8,0,0.26189920902252195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,1,128,1,fp8,fp8,0,0.07058719992637634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,2,128,1,float16,float16,0,0.07452160120010376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,2,128,1,float16,fp8,0,0.07086719870567322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,96,4,128,1,float16,float16,0,1.031710433959961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,2,128,1,fp8,fp8,0,0.07026559710502625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,4,128,1,float16,float16,0,0.07639679908752442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,4,128,1,float16,fp8,0,0.07149119973182679
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,4,128,1,fp8,fp8,0,0.07110080122947693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,8,128,1,float16,float16,0,0.07753120064735412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,8,128,1,float16,fp8,0,0.07199040055274963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,8,128,1,fp8,fp8,0,0.07218719720840454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,96,128,1,fp8,fp8,0,0.6239327907562255
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,2,128,1,float16,float16,0,0.5154111862182618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,96,4,128,1,fp8,fp8,0,0.1361855983734131
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,96,8,128,1,fp8,fp8,0,0.5116975784301758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,96,1,128,1,float16,fp8,0,0.06966879963874817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,96,1,128,1,float16,float16,0,5.718619155883789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,96,1,128,1,float16,fp8,0,5.801150512695313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,96,1,128,1,fp8,fp8,0,5.7534736633300785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,96,2,128,1,float16,float16,0,5.686796951293945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,96,2,128,1,float16,fp8,0,5.794998550415039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,96,2,128,1,fp8,fp8,0,5.783377456665039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,96,4,128,1,float16,float16,0,5.846231842041016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,96,128,1,float16,float16,0,4.292025756835938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,96,4,128,1,float16,fp8,0,5.884328079223633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,96,128,1,float16,fp8,0,3.9143089294433593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,96,4,128,1,fp8,fp8,0,5.8365119934082035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,96,8,128,1,float16,float16,0,6.020364761352539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,96,8,128,1,float16,fp8,0,5.877969741821289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,96,8,128,1,fp8,fp8,0,5.92703857421875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,96,128,1,fp8,fp8,0,3.928084945678711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,1,128,1,float16,float16,0,2.844868850708008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,1,128,1,float16,fp8,0,2.9292831420898438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,1,128,1,fp8,fp8,0,2.9096912384033202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,2,128,1,float16,float16,0,2.8181951522827147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,2,128,1,float16,fp8,0,2.915523147583008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,2,128,1,fp8,fp8,0,2.8860687255859374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,4,128,1,float16,fp8,0,2.940292739868164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,4,128,1,fp8,fp8,0,2.9461648941040037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,96,128,1,float16,fp8,0,1.9487632751464843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,8,128,1,float16,float16,0,2.971455955505371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,8,128,1,float16,fp8,0,2.9701488494873045
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,8,128,1,fp8,fp8,0,2.9902143478393555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,96,128,1,fp8,fp8,0,1.9700319290161132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,96,4,128,1,float16,float16,0,2.8801679611206055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,1,128,1,float16,float16,0,1.4248703956604003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,1,128,1,float16,fp8,0,1.4455488204956055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,1,128,1,fp8,fp8,0,1.4854207992553712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,2,128,1,float16,float16,0,1.4198224067687988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,2,128,1,float16,fp8,0,1.4748527526855468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,2,128,1,fp8,fp8,0,1.4691247940063477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,4,128,1,float16,float16,0,1.4414608001708984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,4,128,1,float16,fp8,0,1.480292797088623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,4,128,1,fp8,fp8,0,1.4580672264099122
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,96,128,1,float16,float16,0,2.158883285522461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,8,128,1,float16,float16,0,1.4754544258117677
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,8,128,1,float16,fp8,0,1.488225555419922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,96,8,128,1,fp8,fp8,0,1.4991776466369628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,96,128,1,float16,fp8,0,0.9925168037414551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,1,128,1,float16,float16,0,0.7084239959716797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,1,128,1,fp8,fp8,0,0.7392528057098389
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,1,128,1,float16,fp8,0,0.7340223789215088
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,2,128,1,float16,float16,0,0.7161935806274414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,2,128,1,float16,fp8,0,0.7403647899627686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,2,128,1,fp8,fp8,0,0.7392255783081054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,4,128,1,float16,float16,0,0.718617582321167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,4,128,1,float16,fp8,0,0.7453792095184326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,4,128,1,fp8,fp8,0,0.7392176151275635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,8,128,1,float16,float16,0,0.7406479835510253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,8,128,1,float16,fp8,0,0.7547167778015137
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,96,128,1,float16,float16,0,0.5505199909210206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,8,128,1,fp8,fp8,0,0.754697608947754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,1,128,1,float16,float16,0,0.36245439052581785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,96,128,1,float16,fp8,0,0.5036015987396241
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,1,128,1,float16,fp8,0,0.3750495910644531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,1,128,1,fp8,fp8,0,0.36872799396514894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,2,128,1,float16,float16,0,0.3667167901992798
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,2,128,1,float16,fp8,0,0.3732032060623169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,2,128,1,fp8,fp8,0,0.3752016067504883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,4,128,1,float16,float16,0,0.37097439765930174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,4,128,1,float16,fp8,0,0.3715183973312378
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,4,128,1,fp8,fp8,0,0.37796640396118164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,8,128,1,float16,float16,0,0.3771087884902954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,8,128,1,float16,fp8,0,0.3822848081588745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,8,128,1,fp8,fp8,0,0.3837552070617676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,96,128,1,float16,float16,0,0.2838639974594116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,96,128,1,fp8,fp8,0,0.2578847885131836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,1,128,1,fp8,fp8,0,0.18955999612808228
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,2,128,1,float16,float16,0,0.18844159841537475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,2,128,1,float16,fp8,0,0.19020160436630248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,2,128,1,fp8,fp8,0,0.19181760549545288
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,4,128,1,float16,float16,0,0.18947839736938477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,4,128,1,float16,fp8,0,0.19256800413131714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,4,128,1,fp8,fp8,0,0.1919424057006836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,8,128,1,float16,float16,0,0.1949903964996338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,8,128,1,float16,fp8,0,0.1941424012184143
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,8,128,1,fp8,fp8,0,0.19480799436569213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,96,128,1,float16,float16,0,0.15032000541687013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,96,128,1,float16,fp8,0,0.13589919805526735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,96,128,1,fp8,fp8,0,0.13542399406433106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,1,128,1,float16,float16,0,0.10020159482955933
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,1,128,1,float16,fp8,0,0.10038559436798096
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,2,128,1,float16,float16,0,0.09943360090255737
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,2,128,1,float16,fp8,0,0.10033600330352783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,2,128,1,fp8,fp8,0,0.10055840015411377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,4,128,1,float16,float16,0,0.1013375997543335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,96,128,1,float16,float16,0,1.0843055725097657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,96,128,1,float16,fp8,0,0.2592832088470459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,1,128,1,float16,float16,0,0.18882880210876465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,4,128,1,float16,fp8,0,0.1024623990058899
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,96,1,128,1,float16,fp8,0,0.19298559427261353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,8,128,1,float16,float16,0,0.10351040363311767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,96,96,128,1,fp8,fp8,0,0.9937392234802246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,8,128,1,fp8,fp8,0,0.1022752046585083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,96,128,1,float16,fp8,0,0.07490400075912476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,96,128,1,fp8,fp8,0,0.0742576003074646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,1,128,1,float16,float16,0,0.05659840106964111
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,1,128,1,float16,fp8,0,0.052692800760269165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,1,128,1,fp8,fp8,0,0.05275200009346008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,2,128,1,float16,float16,0,0.05592160224914551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,2,128,1,float16,fp8,0,0.053337597846984865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,2,128,1,fp8,fp8,0,0.05307040214538574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,4,128,1,float16,float16,0,0.05749120116233826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,4,128,1,float16,fp8,0,0.05336160063743591
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,4,128,1,fp8,fp8,0,0.053390401601791385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,8,128,1,float16,float16,0,0.05802080035209656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,8,128,1,float16,fp8,0,0.054688000679016115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,8,128,1,fp8,fp8,0,0.05494239926338196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,96,128,1,float16,float16,0,0.049116799235343934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,96,128,1,fp8,fp8,0,0.042664000391960145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,1,128,1,float16,float16,0,0.029868799448013305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,1,128,1,float16,fp8,0,0.03115360140800476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,1,128,1,fp8,fp8,0,0.031167998909950256
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,96,96,128,1,fp8,fp8,0,0.5011648178100586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,2,128,1,float16,float16,0,0.030323201417922975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,2,128,1,float16,fp8,0,0.030958399176597595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,2,128,1,fp8,fp8,0,0.03135200142860413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,4,128,1,float16,float16,0,0.030177599191665648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,4,128,1,float16,fp8,0,0.0313647985458374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,4,128,1,fp8,fp8,0,0.03086720108985901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,8,128,1,float16,float16,0,0.030720001459121703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,8,128,1,float16,fp8,0,0.031092798709869383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,8,128,1,fp8,fp8,0,0.031115201115608216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,4,128,1,fp8,fp8,0,0.10108319520950318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,8,128,1,float16,fp8,0,0.10300639867782593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,96,96,128,1,float16,float16,0,0.08218880295753479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,96,1,128,1,float16,float16,0,2.1603599548339845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,96,96,128,1,float16,fp8,0,0.04308159947395325
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,96,1,128,1,fp8,fp8,0,0.10035200119018554
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,96,1,128,1,fp8,fp8,0,2.3347631454467774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,96,2,128,1,float16,float16,0,2.1795040130615235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,96,2,128,1,float16,fp8,0,2.3354143142700194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,96,2,128,1,fp8,fp8,0,2.3144432067871095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,96,4,128,1,float16,float16,0,2.2188016891479494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,96,4,128,1,float16,fp8,0,2.351425552368164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,96,4,128,1,fp8,fp8,0,2.3537744522094726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,96,1,128,1,float16,fp8,0,2.326568031311035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,96,128,1,float16,fp8,0,1.6227584838867188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,96,128,1,float16,float16,0,1.8900224685668945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,1,128,1,float16,float16,0,1.0938608169555664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,96,8,128,1,fp8,fp8,0,2.3625776290893556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,1,128,1,float16,fp8,0,1.1735088348388671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,1,128,1,fp8,fp8,0,1.1717951774597168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,2,128,1,float16,float16,0,1.0939167976379394
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,2,128,1,float16,fp8,0,1.174846363067627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,2,128,1,fp8,fp8,0,1.16702880859375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,4,128,1,float16,float16,0,1.1184176445007323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,96,8,128,1,float16,float16,0,2.278424072265625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,96,8,128,1,float16,fp8,0,2.381390380859375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,4,128,1,float16,fp8,0,1.1828096389770508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,4,128,1,fp8,fp8,0,1.1735263824462892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,8,128,1,float16,float16,0,1.15185604095459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,8,128,1,float16,fp8,0,1.1891663551330567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,8,128,1,fp8,fp8,0,1.1970911979675294
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,1,128,1,float16,float16,0,0.5508927822113037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,96,128,1,float16,float16,0,0.9563695907592773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,1,128,1,float16,fp8,0,0.5936272144317627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,96,128,1,float16,fp8,0,0.8201999664306641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,96,128,1,fp8,fp8,0,0.8195119857788086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,2,128,1,float16,float16,0,0.5571360111236572
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,96,96,128,1,fp8,fp8,0,1.6228687286376953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,2,128,1,float16,fp8,0,0.5883088111877441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,4,128,1,float16,float16,0,0.5676608085632324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,2,128,1,fp8,fp8,0,0.5918064117431641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,4,128,1,fp8,fp8,0,0.5974832057952881
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,8,128,1,float16,float16,0,0.5821616172790527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,8,128,1,float16,fp8,0,0.6055935859680176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,8,128,1,fp8,fp8,0,0.6005231857299804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,96,128,1,float16,float16,0,0.4875951766967773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,96,128,1,float16,fp8,0,0.41582717895507815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,96,128,1,fp8,fp8,0,0.4169600009918213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,1,128,1,float16,fp8,0,0.3012847900390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,2,128,1,float16,fp8,0,0.30127999782562254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,2,128,1,fp8,fp8,0,0.3025631904602051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,4,128,1,float16,float16,0,0.29052960872650146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,4,128,1,float16,fp8,0,0.30360798835754393
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,4,128,1,fp8,fp8,0,0.3042288064956665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,8,128,1,float16,float16,0,0.2982480049133301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,8,128,1,float16,fp8,0,0.3076848030090332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,8,128,1,fp8,fp8,0,0.30955679416656495
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,96,128,1,float16,float16,0,0.25127999782562255
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,96,128,1,float16,fp8,0,0.21497759819030762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,96,128,1,fp8,fp8,0,0.21564159393310547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,1,128,1,float16,float16,0,0.14861119985580445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,1,128,1,float16,fp8,0,0.1568384051322937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,1,128,1,fp8,fp8,0,0.15680320262908937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,1,128,1,fp8,fp8,0,0.5883471965789795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,2,128,1,float16,float16,0,0.15057120323181153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,2,128,1,float16,fp8,0,0.15734080076217652
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,4,128,1,float16,float16,0,0.1527616024017334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,4,128,1,float16,fp8,0,0.15823999643325806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,2,128,1,fp8,fp8,0,0.15750080347061157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,4,128,1,fp8,fp8,0,0.15877439975738525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,96,4,128,1,float16,fp8,0,0.5992015838623047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,8,128,1,fp8,fp8,0,0.16103839874267578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,96,128,1,float16,float16,0,0.13527840375900269
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,96,128,1,float16,fp8,0,0.11573760509490967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,96,128,1,fp8,fp8,0,0.11594560146331787
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,1,128,1,float16,float16,0,0.08151040077209473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,1,128,1,float16,fp8,0,0.08552640080451965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,1,128,1,fp8,fp8,0,0.08560960292816162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,2,128,1,float16,float16,0,0.08161119818687439
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,2,128,1,float16,fp8,0,0.08605120182037354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,2,128,1,fp8,fp8,0,0.0859935998916626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,4,128,1,float16,float16,0,0.08324159979820252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,4,128,1,float16,fp8,0,0.08688960075378419
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,4,128,1,fp8,fp8,0,0.0867471992969513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,8,128,1,float16,float16,0,0.08443999886512757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,8,128,1,float16,fp8,0,0.08751199841499328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,96,8,128,1,fp8,fp8,0,0.08758559823036194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,96,128,1,float16,float16,0,0.07292479872703553
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,96,128,1,float16,fp8,0,0.06457599997520447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,96,128,1,fp8,fp8,0,0.06400480270385742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,1,128,1,float16,fp8,0,0.04556640088558197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,1,128,1,fp8,fp8,0,0.04560000002384186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,2,128,1,float16,float16,0,0.046881601214408875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,2,128,1,float16,fp8,0,0.04579519927501678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,2,128,1,fp8,fp8,0,0.046147200465202334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,4,128,1,float16,float16,0,0.04734239876270294
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,4,128,1,float16,fp8,0,0.04636960029602051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,4,128,1,fp8,fp8,0,0.04625119864940643
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,8,128,1,float16,float16,0,0.15581120252609254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,96,8,128,1,float16,fp8,0,0.16067039966583252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,8,128,1,float16,fp8,0,0.04758720099925995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,8,128,1,fp8,fp8,0,0.04806079864501953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,96,128,1,float16,float16,0,0.04506720006465912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,96,128,1,float16,fp8,0,0.03721440136432648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,96,128,1,fp8,fp8,0,0.03674240112304687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,1,128,1,float16,float16,0,0.02545279860496521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,1,128,1,float16,float16,0,0.28386080265045166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,1,128,1,float16,fp8,0,0.02767840027809143
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,1,128,1,fp8,fp8,0,0.027272000908851624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,2,128,1,float16,float16,0,0.02569279968738556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,2,128,1,fp8,fp8,0,0.027182400226593018
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,4,128,1,float16,float16,0,0.025809600949287415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,4,128,1,float16,fp8,0,0.0272816002368927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,1,128,1,fp8,fp8,0,0.2993920087814331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,4,128,1,fp8,fp8,0,0.027393600344657897
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,8,128,1,float16,float16,0,0.026124799251556398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,8,128,1,float16,fp8,0,0.026976001262664796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,96,128,1,float16,float16,0,0.025278401374816895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,96,128,1,float16,fp8,0,0.022201600670814513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,96,128,1,fp8,fp8,0,0.022433599829673766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,1,128,1,float16,float16,0,0.019088000059127808
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,1,128,1,float16,fp8,0,0.018400000035762788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,1,128,1,fp8,fp8,0,0.018456000089645385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,2,128,1,float16,float16,0,0.018723200261592864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,2,128,1,float16,fp8,0,0.01847680062055588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,2,128,1,fp8,fp8,0,0.018459199368953703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,4,128,1,float16,float16,0,0.01919199973344803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,4,128,1,float16,fp8,0,0.01857600063085556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,4,128,1,fp8,fp8,0,0.018369600176811218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,8,128,1,float16,float16,0,0.019380800426006317
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,1,128,1,float16,float16,0,0.04645760059356689
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,8,128,1,fp8,fp8,0,0.018462400138378143
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,96,1,128,1,float16,float16,0,1.0117183685302735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,96,1,128,1,float16,fp8,0,1.103279972076416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,96,8,128,1,float16,float16,0,0.048600000143051145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,2,128,1,float16,fp8,0,0.027305600047111512
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,96,2,128,1,float16,float16,0,1.01735200881958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,96,2,128,1,float16,float16,0,0.2846400022506714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,96,2,128,1,float16,fp8,0,1.1117440223693849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,96,8,128,1,fp8,fp8,0,0.027412799000740052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,96,2,128,1,fp8,fp8,0,1.1128751754760742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,96,4,128,1,float16,float16,0,1.0244848251342773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,96,8,128,1,float16,fp8,0,0.01847680062055588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,96,4,128,1,fp8,fp8,0,1.1235952377319336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,96,8,128,1,float16,float16,0,1.0578927993774414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,96,8,128,1,float16,fp8,0,1.1402959823608398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,96,8,128,1,fp8,fp8,0,1.141160011291504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,1,128,1,float16,float16,0,0.5130767822265625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,96,128,1,float16,fp8,0,0.8165120124816895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,1,128,1,float16,fp8,0,0.5554927825927735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,96,128,1,fp8,fp8,0,0.8171248435974121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,1,128,1,fp8,fp8,0,0.5561888217926025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,2,128,1,float16,float16,0,0.5144544124603272
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,2,128,1,fp8,fp8,0,0.5600048065185547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,4,128,1,float16,float16,0,0.5232975959777832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,4,128,1,float16,fp8,0,0.5644336223602295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,4,128,1,fp8,fp8,0,0.5649936199188232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,8,128,1,float16,float16,0,0.5356383800506592
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,8,128,1,float16,fp8,0,0.5762159824371338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,8,128,1,fp8,fp8,0,0.5757376194000244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,96,128,1,float16,float16,0,0.45890240669250487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,96,128,1,float16,fp8,0,0.41461758613586425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,96,128,1,fp8,fp8,0,0.4142767906188965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,1,128,1,float16,float16,0,0.26142239570617676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,1,128,1,float16,fp8,0,0.2848479986190796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,96,4,128,1,float16,fp8,0,1.1158576011657715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,2,128,1,float16,fp8,0,0.2861407995223999
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,2,128,1,fp8,fp8,0,0.28543519973754883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,4,128,1,float16,float16,0,0.26729280948638917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,96,128,1,float16,float16,0,0.8976335525512695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,4,128,1,float16,fp8,0,0.2884783983230591
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,4,128,1,fp8,fp8,0,0.28871200084686277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,96,1,128,1,fp8,fp8,0,1.0937600135803223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,8,128,1,float16,float16,0,0.2737663984298706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,96,128,1,float16,float16,0,0.2371840000152588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,96,2,128,1,float16,fp8,0,0.5584479808807373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,96,128,1,float16,fp8,0,0.21487839221954347
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,96,128,1,fp8,fp8,0,0.21400320529937744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,1,128,1,float16,float16,0,0.13949439525604249
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,1,128,1,float16,fp8,0,0.14806079864501953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,1,128,1,fp8,fp8,0,0.14612159729003907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,2,128,1,float16,fp8,0,0.14931999444961547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,2,128,1,fp8,fp8,0,0.1472767949104309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,4,128,1,float16,float16,0,0.13953919410705568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,4,128,1,float16,fp8,0,0.1473744034767151
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,4,128,1,fp8,fp8,0,0.1482367992401123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,8,128,1,float16,float16,0,0.14461920261383057
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,8,128,1,float16,fp8,0,0.15280319452285768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,1,128,1,fp8,fp8,0,0.28462719917297363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,96,128,1,float16,float16,0,0.12939679622650146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,96,128,1,float16,fp8,0,0.11351360082626342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,96,128,1,fp8,fp8,0,0.11263359785079956
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,1,128,1,float16,float16,0,0.077019202709198
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,1,128,1,fp8,fp8,0,0.08013280034065247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,2,128,1,float16,float16,0,0.07633439898490905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,2,128,1,float16,fp8,0,0.0803167998790741
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,2,128,1,fp8,fp8,0,0.08028320074081421
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,4,128,1,float16,float16,0,0.07697439789772034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,8,128,1,float16,fp8,0,0.2948512077331543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,4,128,1,fp8,fp8,0,0.08047199845314026
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,8,128,1,float16,float16,0,0.07796000242233277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,8,128,1,float16,fp8,0,0.08173919916152954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,2,128,1,float16,float16,0,0.13993439674377442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,8,128,1,fp8,fp8,0,0.08154240250587463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,96,128,1,float16,float16,0,0.0704479992389679
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,96,128,1,float16,fp8,0,0.060470402240753174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,1,128,1,float16,float16,0,0.04270879924297333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,96,128,1,fp8,fp8,0,0.06079999804496765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,1,128,1,fp8,fp8,0,0.040761598944664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,2,128,1,float16,float16,0,0.043486401438713074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,2,128,1,float16,fp8,0,0.04082559943199158
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,2,128,1,fp8,fp8,0,0.04104000031948089
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,4,128,1,float16,float16,0,0.043635201454162595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,4,128,1,float16,fp8,0,0.04145599901676178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,4,128,1,fp8,fp8,0,0.04129120111465454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,8,128,1,float16,float16,0,0.044624000787734985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,8,128,1,float16,fp8,0,0.04312640130519867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,8,128,1,fp8,fp8,0,0.04262720048427582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,96,128,1,float16,float16,0,0.04232639968395233
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,2,128,1,float16,float16,0,0.262662410736084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,96,128,1,float16,fp8,0,0.031472000479698184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,96,128,1,fp8,fp8,0,0.0313728004693985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,1,128,1,float16,float16,0,0.02210720032453537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,1,128,1,float16,fp8,0,0.07939519882202148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,2,128,1,float16,float16,0,0.02228959947824478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,2,128,1,float16,fp8,0,0.02399519979953766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,2,128,1,fp8,fp8,0,0.023659199476242065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,4,128,1,float16,float16,0,0.02282879948616028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,4,128,1,float16,fp8,0,0.023951999843120575
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,4,128,1,fp8,fp8,0,0.024059200286865236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,8,128,1,float16,fp8,0,0.024116800725460054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,8,128,1,fp8,fp8,0,0.024145600199699403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,96,128,1,float16,float16,0,0.025737598538398743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,96,128,1,float16,fp8,0,0.02019840031862259
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,96,8,128,1,fp8,fp8,0,0.29414079189300535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,96,128,1,fp8,fp8,0,0.02017440050840378
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,1,128,1,float16,float16,0,0.0173567995429039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,1,128,1,fp8,fp8,0,0.018515199422836304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,2,128,1,float16,float16,0,0.01733279973268509
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,2,128,1,float16,fp8,0,0.01839040070772171
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,2,128,1,fp8,fp8,0,0.01841440051794052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,4,128,1,float16,float16,0,0.017643199861049653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,4,128,1,float16,fp8,0,0.01855040043592453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,4,128,1,fp8,fp8,0,0.018515199422836304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,8,128,1,float16,float16,0,0.017803199589252472
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,8,128,1,float16,fp8,0,0.018537600338459016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,8,128,1,fp8,fp8,0,0.01849119961261749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,96,128,1,float16,float16,0,0.015620799362659454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,96,128,1,float16,fp8,0,0.01475680023431778
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,96,128,1,fp8,fp8,0,0.014535999298095703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,96,8,128,1,fp8,fp8,0,0.1508911967277527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,1,128,1,float16,float16,0,0.012636800110340119
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,1,128,1,float16,fp8,0,0.01340319961309433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,2,128,1,float16,float16,0,0.012652799487113953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,2,128,1,float16,fp8,0,0.013230399787425995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,1,128,1,float16,fp8,0,0.024038399755954742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,2,128,1,fp8,fp8,0,0.013167999684810638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,4,128,1,float16,fp8,0,0.013179199397563934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,4,128,1,fp8,fp8,0,0.013382400572299957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,8,128,1,float16,float16,0,0.012918399274349212
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,8,128,1,float16,float16,0,0.023078399896621703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,8,128,1,float16,fp8,0,0.013443200290203095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,8,128,1,fp8,fp8,0,0.01356479972600937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,96,4,128,1,float16,fp8,0,0.0804207980632782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,96,1,128,1,float16,fp8,0,0.018350400030612946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,96,1,128,1,float16,fp8,0,0.5547984123229981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,96,1,128,1,fp8,fp8,0,0.5525856018066406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,96,1,128,1,float16,fp8,0,0.04086560010910034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,96,2,128,1,float16,float16,0,0.5118656158447266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,96,2,128,1,float16,fp8,0,0.5538832187652588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,96,2,128,1,fp8,fp8,0,0.5575888156890869
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,96,4,128,1,float16,float16,0,0.5212768077850342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,1,128,1,fp8,fp8,0,0.013515199720859527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,96,1,128,1,fp8,fp8,0,0.023947200179100035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,96,4,128,1,float16,float16,0,0.01281599998474121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,96,4,128,1,fp8,fp8,0,0.5614223957061768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,96,8,128,1,float16,fp8,0,0.5770815849304199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,96,8,128,1,fp8,fp8,0,0.5755199909210205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,96,128,1,float16,float16,0,0.6376095771789551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,96,128,1,float16,fp8,0,0.619974422454834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,96,1,128,1,float16,float16,0,0.5085904121398925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,1,128,1,float16,fp8,0,0.2836751937866211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,1,128,1,fp8,fp8,0,0.2834543943405151
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,2,128,1,float16,float16,0,0.26144959926605227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,2,128,1,float16,fp8,0,0.2855407953262329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,2,128,1,fp8,fp8,0,0.2840751886367798
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,4,128,1,float16,fp8,0,0.28736000061035155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,4,128,1,fp8,fp8,0,0.2847743988037109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,8,128,1,float16,float16,0,0.27591838836669924
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,8,128,1,float16,fp8,0,0.2940623998641968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,96,4,128,1,float16,fp8,0,0.557148790359497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,96,128,1,float16,float16,0,0.328110408782959
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,96,128,1,float16,fp8,0,0.3171600103378296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,1,128,1,float16,float16,0,0.13810399770736695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,1,128,1,float16,fp8,0,0.14621599912643432
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,1,128,1,fp8,fp8,0,0.14803999662399292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,2,128,1,float16,float16,0,0.13952319622039794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,2,128,1,float16,fp8,0,0.14791200160980225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,96,128,1,fp8,fp8,0,0.6197679996490478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,1,128,1,float16,float16,0,0.2606544017791748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,4,128,1,float16,float16,0,0.13972320556640624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,4,128,1,float16,fp8,0,0.15149120092391968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,4,128,1,fp8,fp8,0,0.15219199657440186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,8,128,1,float16,float16,0,0.1441823959350586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,8,128,1,float16,fp8,0,0.15507999658584595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,8,128,1,fp8,fp8,0,0.15329920053482055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,96,128,1,float16,float16,0,0.1727776050567627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,96,128,1,float16,fp8,0,0.16719679832458495
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,4,128,1,float16,float16,0,0.2645024061203003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,96,128,1,fp8,fp8,0,0.1663920044898987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,1,128,1,float16,float16,0,0.07701600193977357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,1,128,1,float16,fp8,0,0.08009920120239258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,2,128,1,float16,float16,0,0.07670400142669678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,2,128,1,float16,fp8,0,0.08090400099754333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,2,128,1,fp8,fp8,0,0.08040639758110046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,4,128,1,float16,float16,0,0.07820640206336975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,4,128,1,float16,fp8,0,0.08146880269050598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,4,128,1,fp8,fp8,0,0.0808143973350525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,8,128,1,float16,fp8,0,0.08261439800262452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,8,128,1,fp8,fp8,0,0.08200479745864868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,96,128,1,float16,float16,0,0.09283519983291626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,96,128,1,float16,fp8,0,0.08779039978981018
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,96,128,1,fp8,fp8,0,0.08726080060005188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,1,128,1,float16,float16,0,0.04258559942245484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,1,128,1,float16,fp8,0,0.040838399529457094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,96,8,128,1,fp8,fp8,0,0.2944544076919556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,1,128,1,fp8,fp8,0,0.04092159867286682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,2,128,1,float16,float16,0,0.043196800351142886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,2,128,1,float16,fp8,0,0.040915200114250184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,4,128,1,float16,float16,0,0.04402399957180023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,4,128,1,float16,fp8,0,0.041259199380874634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,4,128,1,fp8,fp8,0,0.04153119921684265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,96,8,128,1,float16,float16,0,0.5367055892944336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,96,128,1,fp8,fp8,0,0.3161504030227661
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,8,128,1,fp8,fp8,0,0.04264639914035797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,96,128,1,float16,float16,0,0.053560000658035276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,96,128,1,float16,fp8,0,0.0455487996339798
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,96,128,1,fp8,fp8,0,0.04596000015735626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,1,128,1,float16,fp8,0,0.024009600281715393
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,1,128,1,fp8,fp8,0,0.02391040027141571
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,2,128,1,float16,float16,0,0.022188800573349
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,2,128,1,float16,fp8,0,0.023843200504779817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,2,128,1,fp8,fp8,0,0.023801599442958832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,4,128,1,float16,float16,0,0.022460800409317017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,4,128,1,float16,fp8,0,0.024001599848270418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,4,128,1,fp8,fp8,0,0.024105599522590636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,8,128,1,float16,float16,0,0.02274879962205887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,1,128,1,fp8,fp8,0,0.07972800135612487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,8,128,1,fp8,fp8,0,0.023889599740505217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,96,128,1,float16,float16,0,0.02738400101661682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,96,8,128,1,float16,float16,0,0.07938079833984375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,96,128,1,float16,fp8,0,0.024372799694538115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,96,128,1,fp8,fp8,0,0.024475200474262236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,1,128,1,float16,fp8,0,0.018199999630451203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,1,128,1,fp8,fp8,0,0.01804639995098114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,2,128,1,float16,float16,0,0.017027199268341064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,2,128,1,float16,fp8,0,0.018318399786949158
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,2,128,1,fp8,fp8,0,0.018243199586868285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,2,128,1,fp8,fp8,0,0.04093759953975677
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,4,128,1,float16,float16,0,0.017348800599575043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,4,128,1,float16,fp8,0,0.018507200479507446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,4,128,1,fp8,fp8,0,0.018270400166511536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,8,128,1,float16,float16,0,0.017529599368572235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,8,128,1,float16,fp8,0,0.042664000391960145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,8,128,1,fp8,fp8,0,0.018287999927997588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,96,128,1,float16,float16,0,0.0189983993768692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,96,128,1,float16,fp8,0,0.01899999976158142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,96,128,1,fp8,fp8,0,0.018995200097560883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,1,128,1,float16,float16,0,0.022060799598693847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,1,128,1,float16,float16,0,0.012643200159072877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,1,128,1,float16,fp8,0,0.013395200669765472
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,1,128,1,fp8,fp8,0,0.013607999682426453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,2,128,1,float16,float16,0,0.012436799705028534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,2,128,1,fp8,fp8,0,0.0133200004696846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,4,128,1,float16,float16,0,0.01276639997959137
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,4,128,1,float16,fp8,0,0.013500800728797913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,4,128,1,fp8,fp8,0,0.013374400138854981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,8,128,1,float16,float16,0,0.012811200320720672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,8,128,1,float16,fp8,0,0.013515199720859527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,8,128,1,fp8,fp8,0,0.013371199369430542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,96,128,1,float16,float16,0,0.014521600306034088
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,96,8,128,1,float16,fp8,0,0.02415039986371994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,96,128,1,float16,fp8,0,0.01398559957742691
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,96,128,1,fp8,fp8,0,0.014156800508499146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,1,128,1,float16,float16,0,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,1,128,1,float16,fp8,0,0.013116799294948578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,1,128,1,fp8,fp8,0,0.013006399571895599
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,1,128,1,float16,float16,0,0.017103999853134155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,2,128,1,fp8,fp8,0,0.01326719969511032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,4,128,1,float16,float16,0,0.012518399953842163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,4,128,1,float16,fp8,0,0.01329600065946579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,4,128,1,fp8,fp8,0,0.013329599797725678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,8,128,1,float16,float16,0,0.0124719999730587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,8,128,1,float16,fp8,0,0.013081599771976472
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,8,128,1,fp8,fp8,0,0.013016000390052795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,96,8,128,1,float16,float16,0,0.044835200905799864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,96,1,128,1,float16,float16,0,0.2596735954284668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,96,1,128,1,float16,fp8,0,0.2846560001373291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,96,1,128,1,fp8,fp8,0,0.28221759796142576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,96,2,128,1,float16,float16,0,0.2647552013397217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,96,2,128,1,float16,fp8,0,0.013116799294948578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,96,2,128,1,float16,fp8,0,0.28264799118041994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,96,2,128,1,fp8,fp8,0,0.28642079830169676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,96,4,128,1,float16,float16,0,0.27017600536346437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,96,4,128,1,float16,fp8,0,0.2854608058929443
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,96,4,128,1,fp8,fp8,0,0.2900160074234009
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,96,8,128,1,float16,float16,0,0.2771791934967041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,96,8,128,1,float16,fp8,0,0.2904335975646973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,96,8,128,1,fp8,fp8,0,0.2953023910522461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,96,128,1,float16,float16,0,0.5079584121704102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,2,128,1,float16,fp8,0,0.013060800731182098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,96,128,1,float16,fp8,0,0.5260447978973388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,1,128,1,float16,float16,0,0.13855680227279663
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,96,128,1,fp8,fp8,0,0.5286399841308593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,1,128,1,float16,fp8,0,0.14675040245056153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,1,128,1,fp8,fp8,0,0.14684000015258789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,2,128,1,float16,float16,0,0.13865599632263184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,2,128,1,float16,fp8,0,0.15086560249328612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,2,128,1,fp8,fp8,0,0.1475167989730835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,4,128,1,float16,fp8,0,0.14731520414352417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,4,128,1,float16,float16,0,0.1391968011856079
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,8,128,1,float16,float16,0,0.14474719762802124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,8,128,1,float16,fp8,0,0.15225919485092163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,8,128,1,fp8,fp8,0,0.1484495997428894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,96,8,128,1,float16,fp8,0,0.01834080070257187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,96,128,1,float16,float16,0,0.26228320598602295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,96,128,1,float16,fp8,0,0.27399680614471433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,1,128,1,float16,fp8,0,0.08068159818649293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,96,128,1,fp8,fp8,0,0.27357120513916017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,1,128,1,fp8,fp8,0,0.08083199858665466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,2,128,1,float16,fp8,0,0.08120800256729126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,2,128,1,fp8,fp8,0,0.08163999915122985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,4,128,1,float16,float16,0,0.07793440222740174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,4,128,1,float16,fp8,0,0.08156319856643676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,4,128,1,fp8,fp8,0,0.08236640095710754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,8,128,1,float16,float16,0,0.07943840026855468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,8,128,1,float16,fp8,0,0.08319360017776489
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,96,2,128,1,fp8,fp8,0,0.1513360023498535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,8,128,1,fp8,fp8,0,0.08354880213737488
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,96,128,1,float16,float16,0,0.13558559417724608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,1,128,1,float16,float16,0,0.043675199151039124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,96,128,1,fp8,fp8,0,0.14323999881744384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,1,128,1,float16,fp8,0,0.04081279933452606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,1,128,1,fp8,fp8,0,0.04137440025806427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,2,128,1,float16,float16,0,0.04336639940738678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,2,128,1,float16,fp8,0,0.04100160002708435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,4,128,1,float16,float16,0,0.04365119934082031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,4,128,1,float16,fp8,0,0.04129120111465454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,4,128,1,fp8,fp8,0,0.041659200191497804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,8,128,1,float16,float16,0,0.04490079879760742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,8,128,1,float16,fp8,0,0.042894399166107176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,8,128,1,fp8,fp8,0,0.04251680076122284
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,96,128,1,float16,float16,0,0.07635040283203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,96,128,1,float16,fp8,0,0.07411680221557618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,96,128,1,fp8,fp8,0,0.07405920028686523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,1,128,1,float16,float16,0,0.022495999932289124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,1,128,1,float16,fp8,0,0.023999999463558196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,1,128,1,fp8,fp8,0,0.02401919960975647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,2,128,1,float16,float16,0,0.022281600534915923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,2,128,1,float16,fp8,0,0.024086399376392363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,2,128,1,fp8,fp8,0,0.023771199584007262
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,4,128,1,float16,float16,0,0.022987200319766997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,4,128,1,float16,fp8,0,0.023897600173950196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,4,128,1,fp8,fp8,0,0.0240447998046875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,8,128,1,float16,float16,0,0.022971199452877046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,8,128,1,float16,fp8,0,0.024035200476646423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,96,8,128,1,fp8,fp8,0,0.024076800048351287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,96,4,128,1,fp8,fp8,0,0.1493391990661621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,96,128,1,float16,float16,0,0.037801599502563475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,96,128,1,fp8,fp8,0,0.038308799266815186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,1,128,1,float16,float16,0,0.017272000014781953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,1,128,1,float16,fp8,0,0.01857759952545166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,1,128,1,fp8,fp8,0,0.018545599281787874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,2,128,1,float16,float16,0,0.07703520059585571
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,2,128,1,float16,fp8,0,0.01839520037174225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,2,128,1,fp8,fp8,0,0.01839199960231781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,4,128,1,float16,float16,0,0.0172447994351387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,4,128,1,float16,fp8,0,0.018452799320220946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,4,128,1,fp8,fp8,0,0.01841759979724884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,96,128,1,float16,fp8,0,0.1422287940979004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,8,128,1,float16,float16,0,0.017449599504470826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,8,128,1,float16,fp8,0,0.018464000523090364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,96,128,1,float16,float16,0,0.022710399329662324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,96,2,128,1,fp8,fp8,0,0.0417279988527298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,96,128,1,float16,fp8,0,0.023455999791622162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,96,128,1,fp8,fp8,0,0.023342399299144743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,1,128,1,float16,float16,0,0.012723200023174286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,1,128,1,float16,fp8,0,0.01329759955406189
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,2,128,1,float16,float16,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,2,128,1,float16,fp8,0,0.013558399677276612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,2,128,1,fp8,fp8,0,0.013630400598049163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,4,128,1,float16,float16,0,0.012680000066757202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,4,128,1,float16,fp8,0,0.013478399813175201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,96,2,128,1,float16,float16,0,0.012350399792194367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,8,128,1,float16,float16,0,0.012734399735927581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,8,128,1,float16,fp8,0,0.01348160058259964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,8,128,1,fp8,fp8,0,0.013457599282264709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,96,128,1,float16,float16,0,0.01828480064868927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,96,128,1,float16,fp8,0,0.018542400002479552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,96,128,1,fp8,fp8,0,0.01855199933052063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,1,128,1,float16,float16,0,0.012478400021791458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,1,128,1,float16,fp8,0,0.013191999495029449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,1,128,1,fp8,fp8,0,0.01326719969511032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,2,128,1,float16,float16,0,0.012494400143623352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,2,128,1,float16,fp8,0,0.01305440068244934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,2,128,1,float16,float16,0,0.01729599982500076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,4,128,1,float16,float16,0,0.012647999823093415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,4,128,1,float16,fp8,0,0.01310880035161972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,4,128,1,fp8,fp8,0,0.013115200400352477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,8,128,1,float16,float16,0,0.01249919980764389
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,8,128,1,float16,fp8,0,0.013100799918174744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,8,128,1,fp8,fp8,0,0.013251200318336487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,8,128,1,fp8,fp8,0,0.018592000007629395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,96,128,1,float16,float16,0,0.014496000111103058
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,96,128,1,float16,fp8,0,0.014097599685192109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,1,128,1,float16,float16,0,0.012217599898576736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,1,128,1,float16,fp8,0,0.012886400520801543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,1,128,1,fp8,fp8,0,0.013499200344085693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,2,128,1,float16,float16,0,0.012350399792194367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,2,128,1,float16,fp8,0,0.01282079964876175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,2,128,1,fp8,fp8,0,0.012899200618267059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,4,128,1,float16,float16,0,0.012097600102424621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,4,128,1,float16,fp8,0,0.012812800705432892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,8,128,1,float16,float16,0,0.012147200107574464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,96,4,128,1,fp8,fp8,0,0.013264000415802002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,8,128,1,float16,fp8,0,0.013078400492668152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,8,128,1,fp8,fp8,0,0.012990400195121765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,96,1,128,1,float16,float16,0,0.13915679454803467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,96,1,128,1,float16,fp8,0,0.14902880191802978
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,96,1,128,1,fp8,fp8,0,0.1511904001235962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,96,2,128,1,float16,float16,0,0.13801920413970947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,96,2,128,1,float16,fp8,0,0.15274720191955565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,96,2,128,1,fp8,fp8,0,0.1502527952194214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,96,4,128,1,float16,float16,0,0.14181920289993286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,96,1,128,1,float16,float16,0,0.07656959891319275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,96,2,128,1,fp8,fp8,0,0.013041600584983826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,96,8,128,1,float16,float16,0,0.17004799842834473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,96,8,128,1,float16,fp8,0,0.18535200357437134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,96,8,128,1,fp8,fp8,0,0.1835487961769104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,96,128,1,float16,float16,0,0.4465904235839844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,96,128,1,fp8,fp8,0,0.014131200313568116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,1,128,1,float16,float16,0,0.07695680260658264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,1,128,1,fp8,fp8,0,0.012878400087356568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,1,128,1,float16,fp8,0,0.08151519894599915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,96,128,1,fp8,fp8,0,0.4939216136932373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,1,128,1,fp8,fp8,0,0.08193759918212891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,2,128,1,float16,float16,0,0.07767199873924255
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,2,128,1,float16,fp8,0,0.08223199844360352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,2,128,1,fp8,fp8,0,0.0824895977973938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,96,4,128,1,fp8,fp8,0,0.013102400302886962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,4,128,1,float16,fp8,0,0.08256000280380249
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,8,128,1,float16,float16,0,0.09328799843788146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,8,128,1,float16,fp8,0,0.10044640302658081
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,8,128,1,fp8,fp8,0,0.10065280199050904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,96,128,1,float16,float16,0,0.2297215938568115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,96,128,1,float16,fp8,0,0.2536736011505127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,1,128,1,float16,float16,0,0.044331198930740355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,96,128,1,fp8,fp8,0,0.2543951988220215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,1,128,1,float16,fp8,0,0.041345599293708804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,1,128,1,fp8,fp8,0,0.04206719994544983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,2,128,1,float16,float16,0,0.04450399875640869
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,2,128,1,float16,fp8,0,0.042096000909805295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,96,4,128,1,float16,fp8,0,0.14905760288238526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,2,128,1,fp8,fp8,0,0.04164640009403229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,4,128,1,float16,float16,0,0.045332801342010495
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,4,128,1,float16,fp8,0,0.04197440147399902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,4,128,1,fp8,fp8,0,0.04238240122795105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,8,128,1,float16,float16,0,0.053611201047897336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,8,128,1,float16,fp8,0,0.052236801385879515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,96,8,128,1,fp8,fp8,0,0.05170559883117676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,96,128,1,float16,float16,0,0.12428799867630005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,96,128,1,float16,fp8,0,0.13068640232086182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,96,128,1,fp8,fp8,0,0.1314288020133972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,1,128,1,float16,fp8,0,0.024222399294376373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,96,128,1,float16,fp8,0,0.49121599197387694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,1,128,1,fp8,fp8,0,0.024217599630355836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,2,128,1,float16,float16,0,0.022385600209236144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,2,128,1,fp8,fp8,0,0.024307200312614442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,4,128,1,float16,float16,0,0.022625599801540375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,4,128,1,float16,fp8,0,0.024271999299526215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,4,128,1,fp8,fp8,0,0.024320000410079957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,4,128,1,float16,float16,0,0.07831519842147827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,8,128,1,float16,float16,0,0.026256000995635985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,96,4,128,1,fp8,fp8,0,0.08258559703826904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,8,128,1,float16,fp8,0,0.029518398642539977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,8,128,1,fp8,fp8,0,0.028779199719429015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,96,128,1,float16,float16,0,0.061990398168563846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,96,128,1,float16,fp8,0,0.06606079936027527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,1,128,1,float16,float16,0,0.017347200214862822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,1,128,1,float16,fp8,0,0.01858240067958832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,1,128,1,fp8,fp8,0,0.01844319999217987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,2,128,1,float16,float16,0,0.017214399576187134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,2,128,1,float16,fp8,0,0.018331199884414673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,2,128,1,fp8,fp8,0,0.01860159933567047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,4,128,1,float16,float16,0,0.017260800302028655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,4,128,1,float16,fp8,0,0.01857600063085556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,4,128,1,fp8,fp8,0,0.018385599553585052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,8,128,1,float16,float16,0,0.017318400740623473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,8,128,1,float16,fp8,0,0.01855199933052063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,8,128,1,fp8,fp8,0,0.018606400489807128
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,96,128,1,float16,float16,0,0.03419679999351501
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,96,96,128,1,float16,fp8,0,0.037985599040985106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,96,128,1,float16,fp8,0,0.037452799081802365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,96,128,1,fp8,fp8,0,0.03749600052833557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,1,128,1,float16,float16,0,0.012544000148773193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,1,128,1,fp8,fp8,0,0.013521599769592284
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,2,128,1,float16,float16,0,0.01268640011548996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,2,128,1,float16,fp8,0,0.013659200072288514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,2,128,1,fp8,fp8,0,0.01313920021057129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,4,128,1,float16,float16,0,0.012984000146389008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,96,4,128,1,fp8,fp8,0,0.1534127950668335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,4,128,1,fp8,fp8,0,0.01356319934129715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,8,128,1,float16,float16,0,0.012839999794960023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,8,128,1,fp8,fp8,0,0.01329759955406189
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,96,128,1,float16,float16,0,0.021990400552749634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,96,128,1,float16,fp8,0,0.02292799949645996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,96,128,1,fp8,fp8,0,0.022913600504398345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,1,128,1,float16,float16,0,0.022444799542427063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,1,128,1,float16,float16,0,0.012483199685811996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,1,128,1,float16,fp8,0,0.013288000226020813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,2,128,1,float16,float16,0,0.012563200294971466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,2,128,1,float16,fp8,0,0.013356800377368926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,96,2,128,1,float16,fp8,0,0.024222399294376373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,2,128,1,fp8,fp8,0,0.013312000036239623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,4,128,1,float16,float16,0,0.012435200065374375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,4,128,1,float16,fp8,0,0.013425600528717042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,4,128,1,fp8,fp8,0,0.013203200697898865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,8,128,1,float16,float16,0,0.012443199753761292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,8,128,1,float16,fp8,0,0.01316159963607788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,8,128,1,fp8,fp8,0,0.01305440068244934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,96,128,1,float16,float16,0,0.018169599771499633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,96,128,1,float16,fp8,0,0.018177600204944612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,96,128,1,fp8,fp8,0,0.018292799592018127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,1,128,1,float16,float16,0,0.01213119998574257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,1,128,1,float16,fp8,0,0.012929600477218629
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,1,128,1,fp8,fp8,0,0.013052800297737121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,2,128,1,float16,float16,0,0.01244800016283989
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,2,128,1,float16,fp8,0,0.013051199913024902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,2,128,1,fp8,fp8,0,0.013079999387264252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,96,96,128,1,fp8,fp8,0,0.06539199948310852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,4,128,1,float16,fp8,0,0.013199999928474426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,4,128,1,fp8,fp8,0,0.013120000064373017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,8,128,1,float16,fp8,0,0.013187199831008911
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,8,128,1,fp8,fp8,0,0.013166399300098419
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,96,128,1,float16,float16,0,0.01430879980325699
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,96,128,1,float16,fp8,0,0.013916799426078796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,96,128,1,fp8,fp8,0,0.013910399377346038
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,1,128,1,float16,float16,0,0.011961600184440613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,1,128,1,float16,fp8,0,0.013436800241470337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,1,128,1,fp8,fp8,0,0.012851199507713318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,2,128,1,float16,float16,0,0.01215519979596138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,4,128,1,float16,fp8,0,0.013475200533866883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,2,128,1,float16,fp8,0,0.012835200130939483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,2,128,1,fp8,fp8,0,0.012697599828243256
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,4,128,1,float16,fp8,0,0.012800000607967377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,4,128,1,fp8,fp8,0,0.01282079964876175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,8,128,1,float16,float16,0,0.012198399752378464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,8,128,1,float16,fp8,0,0.01276479959487915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,8,128,1,fp8,fp8,0,0.01273760050535202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,96,1,128,1,float16,float16,0,0.021171200275421142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,96,1,128,1,fp8,fp8,0,0.013027200102806091
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,96,1,128,1,float16,fp8,0,0.02287680059671402
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,96,1,128,1,fp8,fp8,0,0.022942399978637694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,96,2,128,1,float16,float16,0,0.028275200724601747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,96,2,128,1,float16,fp8,0,0.031518399715423584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,96,2,128,1,fp8,fp8,0,0.03163520097732544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,96,4,128,1,float16,float16,0,0.04307039976119995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,96,4,128,1,float16,fp8,0,0.0495743989944458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,96,4,128,1,fp8,fp8,0,0.04928799867630005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,96,8,128,1,float16,float16,0,0.07169439792633056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,96,8,128,1,float16,fp8,0,0.08481600284576415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,96,8,128,1,fp8,fp8,0,0.08465920090675354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,4,128,1,float16,float16,0,0.012305600196123123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,96,128,1,float16,float16,0,0.3515023946762085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,96,8,128,1,float16,float16,0,0.012270399928092956
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,96,128,1,float16,fp8,0,0.439635181427002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,1,128,1,fp8,fp8,0,0.015705600380897522
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,2,128,1,float16,float16,0,0.018596799671649934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,96,128,1,fp8,fp8,0,0.43715682029724123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,2,128,1,float16,fp8,0,0.020156799256801604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,2,128,1,fp8,fp8,0,0.020316800475120543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,4,128,1,float16,float16,0,0.02616640031337738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,4,128,1,float16,fp8,0,0.029284799098968507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,4,128,1,fp8,fp8,0,0.02928000092506409
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,1,128,1,float16,fp8,0,0.012596799433231354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,8,128,1,float16,fp8,0,0.04681439995765686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,8,128,1,fp8,fp8,0,0.04659680128097534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,96,128,1,float16,float16,0,0.18483519554138184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,96,8,128,1,float16,fp8,0,0.013352000713348388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,96,128,1,fp8,fp8,0,0.22532799243927001
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,1,128,1,float16,fp8,0,0.0141744002699852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,1,128,1,fp8,fp8,0,0.014123199880123139
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,2,128,1,float16,float16,0,0.01342879980802536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,2,128,1,float16,fp8,0,0.014263999462127686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,2,128,1,fp8,fp8,0,0.014215999841690063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,4,128,1,float16,float16,0,0.0172447994351387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,4,128,1,float16,fp8,0,0.018796800076961516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,4,128,1,fp8,fp8,0,0.018721599876880646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,8,128,1,float16,float16,0,0.024408000707626342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,8,128,1,float16,fp8,0,0.027459201216697694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,8,128,1,fp8,fp8,0,0.027599999308586122
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,96,128,1,float16,float16,0,0.10048320293426513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,96,128,1,float16,fp8,0,0.11936639547348023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,1,128,1,float16,float16,0,0.01290079951286316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,1,128,1,float16,float16,0,0.01494079977273941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,1,128,1,float16,fp8,0,0.01353919953107834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,1,128,1,float16,fp8,0,0.015806399285793304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,2,128,1,float16,float16,0,0.012601600587368011
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,2,128,1,float16,fp8,0,0.013435199856758118
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,2,128,1,fp8,fp8,0,0.013436800241470337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,4,128,1,float16,float16,0,0.012596799433231354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,4,128,1,float16,fp8,0,0.01340160071849823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,4,128,1,fp8,fp8,0,0.01353600025177002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,8,128,1,float16,float16,0,0.016715200245380403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,8,128,1,float16,fp8,0,0.01788640022277832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,8,128,1,fp8,fp8,0,0.018087999522686006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,96,8,128,1,float16,float16,0,0.04069119989871979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,96,128,1,float16,float16,0,0.055979198217391966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,96,128,1,float16,fp8,0,0.06416000127792358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,1,128,1,float16,float16,0,0.012078399956226348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,1,128,1,float16,fp8,0,0.012561599910259246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,1,128,1,fp8,fp8,0,0.013035200536251068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,2,128,1,float16,float16,0,0.011865600198507308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,2,128,1,float16,fp8,0,0.013265599310398103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,2,128,1,fp8,fp8,0,0.01273919939994812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,4,128,1,float16,float16,0,0.012289600074291229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,4,128,1,float16,fp8,0,0.012593600153923034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,4,128,1,fp8,fp8,0,0.013177600502967835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,8,128,1,float16,float16,0,0.011963199824094772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,8,128,1,float16,fp8,0,0.013191999495029449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,8,128,1,fp8,fp8,0,0.012996800243854523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,96,128,1,float16,float16,0,0.03335199952125549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,96,128,1,float16,fp8,0,0.03686240017414093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,96,128,1,fp8,fp8,0,0.036643201112747194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,1,128,1,float16,float16,0,0.011902400106191636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,1,128,1,float16,fp8,0,0.01268800050020218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,2,128,1,float16,float16,0,0.012015999853610992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,2,128,1,float16,fp8,0,0.01284320056438446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,2,128,1,fp8,fp8,0,0.012748800218105316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,4,128,1,float16,float16,0,0.01188800036907196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,4,128,1,float16,fp8,0,0.01273760050535202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,4,128,1,fp8,fp8,0,0.012744000554084778
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,8,128,1,float16,float16,0,0.012065599858760833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,8,128,1,float16,fp8,0,0.012848000228404998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,8,128,1,fp8,fp8,0,0.012907199561595917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,96,128,1,float16,float16,0,0.021809600293636322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,96,128,1,float16,fp8,0,0.022819200158119203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,96,128,1,fp8,fp8,0,0.02281759977340698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,1,128,1,float16,float16,0,0.01180960014462471
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,96,128,1,fp8,fp8,0,0.11923040151596069
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,1,128,1,float16,fp8,0,0.012318400293588638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,1,128,1,fp8,fp8,0,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,96,1,128,1,fp8,fp8,0,0.013500800728797913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,2,128,1,float16,fp8,0,0.01223199963569641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,2,128,1,fp8,fp8,0,0.012387199699878693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,4,128,1,float16,float16,0,0.011788800358772278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,4,128,1,float16,fp8,0,0.012454400211572647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,4,128,1,fp8,fp8,0,0.012468799948692322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,8,128,1,float16,float16,0,0.011919999867677689
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,8,128,1,float16,fp8,0,0.012654399871826172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,8,128,1,fp8,fp8,0,0.01260959953069687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,96,128,1,float16,float16,0,0.017969599366188048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,96,128,1,float16,fp8,0,0.017948800325393678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,96,128,1,fp8,fp8,0,0.018119999766349794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,96,96,128,1,fp8,fp8,0,0.06385440230369568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,1,128,1,float16,float16,0,0.011532799899578094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,1,128,1,float16,fp8,0,0.012353599816560746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,1,128,1,fp8,fp8,0,0.01236959993839264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,2,128,1,float16,float16,0,0.011547199636697768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,2,128,1,float16,fp8,0,0.012302400171756744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,96,128,1,float16,fp8,0,0.22441439628601073
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,2,128,1,fp8,fp8,0,0.012307199835777282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,4,128,1,float16,float16,0,0.011428800225257874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,96,1,128,1,float16,float16,0,0.013766400516033173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,8,128,1,float16,fp8,0,0.012531200051307678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,8,128,1,fp8,fp8,0,0.012503999471664428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,96,128,1,float16,float16,0,0.014142400026321411
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,96,1,128,1,fp8,fp8,0,0.012664000689983367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,96,128,1,fp8,fp8,0,0.013811199367046357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,1,128,1,float16,float16,0,0.011457599699497223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,1,128,1,float16,fp8,0,0.0122079998254776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,1,128,1,fp8,fp8,0,0.01220960021018982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,2,128,1,float16,float16,0,0.011358399689197541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,2,128,1,float16,fp8,0,0.012120000272989272
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,2,128,1,fp8,fp8,0,0.01225920021533966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,4,128,1,float16,float16,0,0.011580800265073776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,4,128,1,float16,fp8,0,0.01228479966521263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,4,128,1,fp8,fp8,0,0.012148799747228623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,8,128,1,float16,float16,0,0.011363200098276138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,8,128,1,float16,fp8,0,0.012150400131940842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,8,128,1,fp8,fp8,0,0.012318400293588638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,96,2,128,1,float16,float16,0,0.011664000153541566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,96,4,128,1,float16,float16,0,0.012187200039625168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,4,128,1,fp8,fp8,0,0.01223680004477501
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,96,96,128,1,float16,fp8,0,0.013937599956989288
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,4,128,1,float16,fp8,0,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,96,8,128,1,float16,float16,0,0.011659199744462967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,64,1,128,1,float16,fp8,0,52.672607421875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,64,1,128,1,fp8,fp8,0,52.1251220703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,64,2,128,1,fp8,fp8,0,52.1118408203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,64,2,128,1,float16,fp8,0,52.3412353515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,64,1,128,1,float16,float16,0,61.249334716796874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,64,4,128,1,float16,fp8,0,52.12347412109375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,64,2,128,1,float16,float16,0,62.99378662109375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,64,4,128,1,float16,float16,0,63.293798828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,1,128,1,float16,float16,0,31.665298461914062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,64,4,128,1,fp8,fp8,0,53.01243286132812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,64,8,128,1,float16,fp8,0,52.27646484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,64,8,128,1,fp8,fp8,0,52.640771484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,64,8,128,1,float16,float16,0,62.9608154296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,1,128,1,float16,fp8,0,26.393817138671874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,1,128,1,fp8,fp8,0,26.425979614257812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,2,128,1,float16,fp8,0,26.293032836914062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,2,128,1,float16,float16,0,31.117010498046874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,2,128,1,fp8,fp8,0,25.58849182128906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,4,128,1,float16,float16,0,31.89360656738281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,4,128,1,float16,fp8,0,26.655645751953124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,4,128,1,fp8,fp8,0,26.273355102539064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,8,128,1,float16,fp8,0,26.540939331054688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,8,128,1,float16,float16,0,31.615045166015626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,1,128,1,float16,float16,0,15.263847351074219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,8,128,1,fp8,fp8,0,26.341128540039062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,1,128,1,float16,fp8,0,12.949652099609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,1,128,1,fp8,fp8,0,13.364259338378906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,2,128,1,float16,fp8,0,13.097402954101563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,2,128,1,float16,float16,0,15.601980590820313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,2,128,1,fp8,fp8,0,13.134628295898438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,4,128,1,float16,float16,0,15.711427307128906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,4,128,1,float16,fp8,0,13.374630737304688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,4,128,1,fp8,fp8,0,12.859516906738282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,64,128,1,float16,fp8,0,13.207057189941406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,64,128,1,fp8,fp8,0,13.235552978515624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,8,128,1,float16,float16,0,16.178053283691405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,8,128,1,float16,fp8,0,13.043603515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,64,128,1,float16,float16,0,7.85352783203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,64,128,1,float16,float16,0,15.992648315429687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,64,128,1,float16,fp8,0,6.787278747558593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,64,128,1,float16,fp8,0,26.32427978515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,64,128,1,fp8,fp8,0,6.762483215332031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,64,8,128,1,fp8,fp8,0,12.8987548828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,64,128,1,fp8,fp8,0,26.670028686523438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,1,128,1,float16,fp8,0,6.418097686767578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,1,128,1,float16,float16,0,7.538715362548828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,1,128,1,fp8,fp8,0,6.478842926025391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,2,128,1,float16,float16,0,7.763966369628906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,2,128,1,float16,fp8,0,6.618984222412109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,2,128,1,fp8,fp8,0,6.500489807128906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,64,64,128,1,float16,float16,0,31.347781372070312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,4,128,1,float16,float16,0,8.319403076171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,4,128,1,float16,fp8,0,6.421701049804687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,4,128,1,fp8,fp8,0,6.721377563476563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,8,128,1,float16,float16,0,7.954467010498047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,8,128,1,fp8,fp8,0,6.462014770507812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,64,8,128,1,float16,fp8,0,6.660469055175781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,64,1,128,1,float16,fp8,0,29.140829467773436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,64,1,128,1,fp8,fp8,0,29.523260498046874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,64,2,128,1,float16,fp8,0,29.4189697265625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,64,1,128,1,float16,float16,0,34.86080627441406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,64,4,128,1,float16,fp8,0,29.285870361328126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,64,2,128,1,fp8,fp8,0,30.22675476074219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,64,2,128,1,float16,float16,0,35.58887939453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,64,4,128,1,float16,float16,0,35.665957641601565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,64,128,1,float16,fp8,0,15.596649169921875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,64,128,1,fp8,fp8,0,15.59088592529297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,64,128,1,float16,float16,0,18.698301696777342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,1,128,1,float16,float16,0,17.561077880859376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,64,4,128,1,fp8,fp8,0,29.694036865234374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,64,8,128,1,float16,fp8,0,29.855551147460936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,64,8,128,1,fp8,fp8,0,29.86860046386719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,64,8,128,1,float16,float16,0,36.01181335449219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,1,128,1,float16,fp8,0,15.009065246582031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,1,128,1,fp8,fp8,0,14.495037841796876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,2,128,1,float16,fp8,0,14.669990539550781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,2,128,1,fp8,fp8,0,14.799775695800781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,2,128,1,float16,float16,0,17.802439880371093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,4,128,1,float16,fp8,0,15.208135986328125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,4,128,1,float16,float16,0,18.159512329101563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,4,128,1,fp8,fp8,0,15.0722412109375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,64,128,1,float16,fp8,0,7.600179290771484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,8,128,1,float16,fp8,0,15.089576721191406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,64,128,1,float16,float16,0,9.333684539794922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,8,128,1,float16,float16,0,18.16862335205078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,64,128,1,fp8,fp8,0,7.811436462402344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,64,8,128,1,fp8,fp8,0,14.947367858886718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,1,128,1,float16,float16,0,8.75336151123047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,1,128,1,float16,fp8,0,7.429169464111328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,1,128,1,fp8,fp8,0,7.27401123046875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,2,128,1,float16,fp8,0,7.404268646240235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,2,128,1,float16,float16,0,9.120967864990234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,2,128,1,fp8,fp8,0,7.374654388427734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,4,128,1,float16,float16,0,9.013539123535157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,4,128,1,float16,fp8,0,7.285622406005859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,4,128,1,fp8,fp8,0,7.3473663330078125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,64,128,1,float16,float16,0,4.545687866210938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,64,128,1,float16,fp8,0,3.8378318786621093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,8,128,1,float16,fp8,0,7.363102722167969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,8,128,1,float16,float16,0,9.012445068359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,64,128,1,fp8,fp8,0,3.8770095825195314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,1,128,1,float16,fp8,0,3.6259792327880858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,1,128,1,float16,float16,0,4.300630569458008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,64,8,128,1,fp8,fp8,0,7.836637115478515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,1,128,1,fp8,fp8,0,3.7237152099609374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,2,128,1,float16,fp8,0,3.667475128173828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,2,128,1,fp8,fp8,0,3.8052433013916014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,4,128,1,float16,fp8,0,3.6426353454589844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,4,128,1,fp8,fp8,0,3.7052639007568358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,8,128,1,float16,float16,0,4.528691101074219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,8,128,1,float16,fp8,0,3.6377918243408205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,8,128,1,fp8,fp8,0,3.699785614013672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,2,128,1,float16,float16,0,4.476982498168946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,64,4,128,1,float16,float16,0,4.367305755615234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,64,1,128,1,float16,fp8,0,20.880288696289064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,64,1,128,1,fp8,fp8,0,20.777346801757812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,64,2,128,1,float16,fp8,0,20.600634765625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,64,2,128,1,fp8,fp8,0,20.91016540527344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,64,1,128,1,float16,float16,0,25.15583190917969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,64,4,128,1,float16,fp8,0,20.62672576904297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,64,2,128,1,float16,float16,0,25.058134460449217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,64,4,128,1,float16,float16,0,24.864524841308594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,64,128,1,float16,fp8,0,10.78584976196289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,64,128,1,fp8,fp8,0,10.991343688964843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,64,128,1,float16,float16,0,13.370590209960938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,1,128,1,float16,float16,0,12.998020935058594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,64,4,128,1,fp8,fp8,0,20.80382537841797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,64,8,128,1,float16,fp8,0,21.09966125488281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,64,8,128,1,fp8,fp8,0,21.032351684570312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,64,8,128,1,float16,float16,0,24.83617248535156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,1,128,1,float16,fp8,0,10.962226867675781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,1,128,1,fp8,fp8,0,10.440996551513672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,2,128,1,float16,float16,0,12.675020599365235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,2,128,1,float16,fp8,0,10.353543853759765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,2,128,1,fp8,fp8,0,10.404379272460938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,4,128,1,float16,fp8,0,10.419419097900391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,4,128,1,float16,float16,0,12.64327392578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,4,128,1,fp8,fp8,0,10.1947998046875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,64,128,1,float16,float16,0,6.833570861816407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,64,128,1,float16,fp8,0,5.5153759002685545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,8,128,1,float16,fp8,0,10.502262115478516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,8,128,1,float16,float16,0,12.539631652832032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,64,128,1,fp8,fp8,0,5.419425582885742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,64,8,128,1,fp8,fp8,0,10.45361099243164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,1,128,1,float16,float16,0,6.0815471649169925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,1,128,1,fp8,fp8,0,5.340558242797852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,2,128,1,float16,fp8,0,5.407972717285157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,2,128,1,float16,float16,0,6.298756790161133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,2,128,1,fp8,fp8,0,5.178587341308594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,1,128,1,float16,fp8,0,5.057928085327148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,4,128,1,float16,fp8,0,5.193460845947266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,4,128,1,float16,float16,0,6.148926544189453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,4,128,1,fp8,fp8,0,5.2043312072753904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,64,128,1,float16,float16,0,3.1710111618041994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,64,128,1,float16,fp8,0,2.6345935821533204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,64,128,1,fp8,fp8,0,2.7064624786376954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,8,128,1,float16,fp8,0,5.281342315673828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,1,128,1,float16,float16,0,3.012112045288086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,1,128,1,float16,fp8,0,2.5365407943725584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,1,128,1,fp8,fp8,0,2.6568016052246093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,2,128,1,float16,float16,0,2.810647964477539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,2,128,1,float16,fp8,0,2.571151924133301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,2,128,1,fp8,fp8,0,2.604380798339844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,4,128,1,float16,fp8,0,2.5450511932373048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,4,128,1,float16,float16,0,3.0403167724609377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,8,128,1,float16,float16,0,6.334044647216797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,4,128,1,fp8,fp8,0,2.5694816589355467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,8,128,1,float16,float16,0,2.9980655670166017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,8,128,1,float16,fp8,0,2.574247932434082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,64,8,128,1,fp8,fp8,0,2.5923696517944337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,64,8,128,1,fp8,fp8,0,5.3465534210205075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,64,1,128,1,float16,fp8,0,26.982696533203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,64,1,128,1,fp8,fp8,0,27.231097412109374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,64,2,128,1,float16,fp8,0,27.593368530273438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,64,2,128,1,fp8,fp8,0,27.4802978515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,64,1,128,1,float16,float16,0,32.06719055175781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,64,2,128,1,float16,float16,0,32.421084594726565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,64,4,128,1,float16,float16,0,31.704037475585938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,64,4,128,1,float16,fp8,0,27.04300537109375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,64,128,1,float16,fp8,0,13.952337646484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,64,128,1,float16,float16,0,16.985081481933594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,64,128,1,fp8,fp8,0,14.270384216308594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,64,4,128,1,fp8,fp8,0,27.332806396484376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,1,128,1,float16,float16,0,17.230349731445312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,64,8,128,1,float16,fp8,0,27.673129272460937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,64,8,128,1,fp8,fp8,0,27.50592956542969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,64,8,128,1,float16,float16,0,32.76231689453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,1,128,1,float16,fp8,0,13.222016906738281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,1,128,1,fp8,fp8,0,13.349931335449218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,2,128,1,float16,float16,0,15.861227416992188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,2,128,1,float16,fp8,0,13.442547607421876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,2,128,1,fp8,fp8,0,13.517800903320312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,4,128,1,float16,fp8,0,13.845266723632813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,4,128,1,float16,float16,0,16.527687072753906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,4,128,1,fp8,fp8,0,13.445166015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,64,128,1,float16,fp8,0,7.115593719482422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,8,128,1,float16,fp8,0,13.6098388671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,64,128,1,float16,float16,0,8.845159912109375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,8,128,1,float16,float16,0,16.712535095214843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,64,128,1,fp8,fp8,0,7.25646743774414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,64,8,128,1,fp8,fp8,0,13.84221954345703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,1,128,1,float16,float16,0,8.001719665527343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,1,128,1,float16,fp8,0,6.746340942382813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,1,128,1,fp8,fp8,0,6.750481414794922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,2,128,1,float16,float16,0,8.117454528808594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,2,128,1,float16,fp8,0,6.77034912109375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,2,128,1,fp8,fp8,0,6.6052001953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,4,128,1,float16,float16,0,8.323200225830078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,4,128,1,float16,fp8,0,6.692993927001953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,64,128,1,float16,float16,0,4.1951904296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,4,128,1,fp8,fp8,0,6.792166137695313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,64,128,1,float16,fp8,0,3.463924789428711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,8,128,1,float16,fp8,0,7.055142211914062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,8,128,1,float16,float16,0,8.33249740600586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,64,128,1,fp8,fp8,0,3.645521545410156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,64,8,128,1,fp8,fp8,0,6.732679748535157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,1,128,1,float16,float16,0,3.9182670593261717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,1,128,1,fp8,fp8,0,3.3249439239501952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,1,128,1,float16,fp8,0,3.439022445678711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,2,128,1,float16,float16,0,3.843598556518555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,2,128,1,float16,fp8,0,3.4178768157958985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,2,128,1,fp8,fp8,0,3.4213680267333983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,4,128,1,float16,fp8,0,3.3195518493652343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,4,128,1,float16,float16,0,4.000296020507813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,4,128,1,fp8,fp8,0,3.413827133178711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,64,128,1,float16,fp8,0,1.7573631286621094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,64,128,1,float16,float16,0,2.058087921142578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,8,128,1,float16,fp8,0,3.2951919555664064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,8,128,1,float16,float16,0,3.984369659423828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,64,8,128,1,fp8,fp8,0,3.4131473541259765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,64,128,1,fp8,fp8,0,1.7427967071533204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,1,128,1,float16,float16,0,2.003067207336426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,1,128,1,fp8,fp8,0,1.7118223190307618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,2,128,1,float16,fp8,0,1.636400032043457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,2,128,1,float16,float16,0,2.0453184127807615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,4,128,1,float16,float16,0,1.8245695114135743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,2,128,1,fp8,fp8,0,1.8719039916992188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,4,128,1,float16,fp8,0,1.6594736099243164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,4,128,1,fp8,fp8,0,1.7841184616088868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,8,128,1,float16,fp8,0,1.6706096649169921
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,8,128,1,float16,float16,0,1.8972400665283202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,8,128,1,fp8,fp8,0,1.807579231262207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,64,1,128,1,float16,fp8,0,1.6577983856201173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,64,1,128,1,fp8,fp8,0,15.460430908203126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,64,1,128,1,float16,fp8,0,15.686419677734374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,64,2,128,1,float16,fp8,0,15.6305419921875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,64,2,128,1,fp8,fp8,0,15.54466552734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,64,4,128,1,float16,fp8,0,15.411868286132812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,64,1,128,1,float16,float16,0,18.71912536621094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,64,2,128,1,float16,float16,0,18.906158447265625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,64,4,128,1,float16,float16,0,18.741607666015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,64,128,1,float16,fp8,0,8.173387145996093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,64,128,1,float16,float16,0,9.876261138916016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,64,128,1,fp8,fp8,0,8.427591705322266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,1,128,1,float16,float16,0,9.433392333984376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,64,4,128,1,fp8,fp8,0,15.83527069091797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,64,8,128,1,float16,fp8,0,15.833253479003906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,64,8,128,1,fp8,fp8,0,16.055877685546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,64,8,128,1,float16,float16,0,19.067124938964845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,1,128,1,float16,fp8,0,7.588684844970703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,1,128,1,fp8,fp8,0,7.779374694824218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,2,128,1,float16,float16,0,9.53546371459961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,2,128,1,float16,fp8,0,7.8801933288574215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,2,128,1,fp8,fp8,0,7.720635223388672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,4,128,1,float16,fp8,0,8.027089691162109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,4,128,1,float16,float16,0,9.367654418945312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,4,128,1,fp8,fp8,0,7.79080810546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,64,128,1,float16,float16,0,5.125999832153321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,64,128,1,float16,fp8,0,4.050779342651367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,8,128,1,float16,float16,0,9.214067077636718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,8,128,1,float16,fp8,0,8.026195526123047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,64,128,1,fp8,fp8,0,4.1330608367919925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,64,8,128,1,fp8,fp8,0,7.857307434082031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,1,128,1,float16,float16,0,4.552799987792969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,1,128,1,fp8,fp8,0,3.812363052368164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,2,128,1,float16,fp8,0,3.8257377624511717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,2,128,1,float16,float16,0,4.589673614501953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,2,128,1,fp8,fp8,0,3.865768051147461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,4,128,1,float16,float16,0,4.697900772094727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,1,128,1,float16,fp8,0,4.209806442260742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,4,128,1,fp8,fp8,0,3.8400543212890623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,4,128,1,float16,fp8,0,3.9740734100341797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,8,128,1,float16,fp8,0,3.9461456298828126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,64,128,1,float16,float16,0,2.5522064208984374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,64,128,1,fp8,fp8,0,2.009752082824707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,1,128,1,float16,fp8,0,1.9275760650634766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,8,128,1,fp8,fp8,0,3.906016159057617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,1,128,1,float16,float16,0,2.1528032302856444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,1,128,1,fp8,fp8,0,1.9543584823608398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,2,128,1,float16,fp8,0,1.8991775512695312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,2,128,1,fp8,fp8,0,1.9146432876586914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,2,128,1,float16,float16,0,2.3157392501831056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,4,128,1,float16,float16,0,2.0988927841186524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,64,128,1,float16,fp8,0,2.0052560806274413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,4,128,1,fp8,fp8,0,1.9369440078735352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,64,128,1,float16,float16,0,1.2295455932617188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,8,128,1,float16,fp8,0,1.9325183868408202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,64,128,1,float16,fp8,0,1.006340789794922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,8,128,1,fp8,fp8,0,2.1045343399047853
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,64,128,1,fp8,fp8,0,1.0275903701782227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,1,128,1,float16,float16,0,1.1109503746032714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,1,128,1,fp8,fp8,0,0.9399663925170898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,1,128,1,float16,fp8,0,0.9658448219299316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,2,128,1,float16,float16,0,1.0379568099975587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,2,128,1,float16,fp8,0,0.951195240020752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,4,128,1,float16,fp8,0,2.076852798461914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,4,128,1,float16,fp8,0,0.959716796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,4,128,1,fp8,fp8,0,1.0611680030822754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,8,128,1,float16,float16,0,1.0131952285766601
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,8,128,1,float16,fp8,0,0.9583632469177246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,8,128,1,fp8,fp8,0,0.9587488174438477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,64,8,128,1,float16,float16,0,2.122350311279297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,2,128,1,fp8,fp8,0,1.0581904411315919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,64,4,128,1,float16,float16,0,1.015500831604004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,64,8,128,1,float16,float16,0,4.6012016296386715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,64,1,128,1,float16,fp8,0,14.295904541015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,64,1,128,1,fp8,fp8,0,14.320321655273437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,64,2,128,1,float16,fp8,0,14.576609802246093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,64,1,128,1,float16,float16,0,17.45752410888672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,64,2,128,1,fp8,fp8,0,14.401594543457032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,64,2,128,1,float16,float16,0,17.49014892578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,64,4,128,1,float16,fp8,0,14.303494262695313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,64,4,128,1,float16,float16,0,17.13145751953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,64,128,1,fp8,fp8,0,7.844652557373047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,64,128,1,float16,fp8,0,7.996572875976563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,64,128,1,float16,float16,0,9.420439910888671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,1,128,1,float16,float16,0,8.384553527832031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,64,4,128,1,fp8,fp8,0,14.512484741210937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,64,8,128,1,float16,fp8,0,14.68497314453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,64,8,128,1,float16,float16,0,17.125140380859374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,64,8,128,1,fp8,fp8,0,14.761276245117188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,1,128,1,float16,fp8,0,7.151767730712891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,1,128,1,fp8,fp8,0,7.305182647705078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,2,128,1,float16,float16,0,8.713409423828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,2,128,1,float16,fp8,0,7.243937683105469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,2,128,1,fp8,fp8,0,7.252519989013672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,4,128,1,float16,fp8,0,7.517559814453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,4,128,1,float16,float16,0,9.522245025634765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,4,128,1,fp8,fp8,0,7.318764495849609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,64,128,1,float16,fp8,0,3.92077751159668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,64,128,1,float16,float16,0,4.765744018554687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,8,128,1,float16,fp8,0,7.303300476074218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,8,128,1,float16,float16,0,9.406575775146484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,1,128,1,float16,fp8,0,3.567127990722656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,64,8,128,1,fp8,fp8,0,7.485688018798828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,64,128,1,fp8,fp8,0,3.8991249084472654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,1,128,1,float16,float16,0,4.189603042602539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,1,128,1,fp8,fp8,0,3.5529296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,2,128,1,float16,fp8,0,3.6298526763916015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,2,128,1,float16,float16,0,4.2121726989746096
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,2,128,1,fp8,fp8,0,3.7033199310302733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,4,128,1,float16,fp8,0,3.608561706542969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,4,128,1,fp8,fp8,0,3.573072052001953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,4,128,1,float16,float16,0,4.3715057373046875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,64,128,1,float16,float16,0,2.3043920516967775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,8,128,1,float16,fp8,0,3.5878177642822267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,8,128,1,float16,float16,0,4.312035369873047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,64,8,128,1,fp8,fp8,0,3.6640705108642577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,64,128,1,float16,fp8,0,1.961827278137207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,64,128,1,fp8,fp8,0,1.9579919815063476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,1,128,1,float16,fp8,0,1.924692726135254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,1,128,1,fp8,fp8,0,1.902280044555664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,2,128,1,float16,float16,0,1.9716127395629883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,2,128,1,float16,fp8,0,1.9217744827270509
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,2,128,1,fp8,fp8,0,1.8049535751342773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,4,128,1,float16,float16,0,1.9695728302001954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,4,128,1,float16,fp8,0,1.8092432022094727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,4,128,1,fp8,fp8,0,1.8096351623535156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,64,128,1,float16,float16,0,1.1074496269226075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,8,128,1,float16,float16,0,1.9986160278320313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,8,128,1,float16,fp8,0,1.9330976486206055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,64,128,1,float16,fp8,0,0.985102367401123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,64,128,1,fp8,fp8,0,1.0145919799804688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,1,128,1,float16,float16,0,1.056276798248291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,1,128,1,float16,fp8,0,1.0025263786315919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,1,128,1,fp8,fp8,0,0.9091168403625488
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,2,128,1,float16,float16,0,0.9510512351989746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,2,128,1,float16,fp8,0,0.9081711769104004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,2,128,1,fp8,fp8,0,1.0349103927612304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,4,128,1,float16,fp8,0,0.897708797454834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,4,128,1,fp8,fp8,0,0.9169055938720703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,8,128,1,float16,float16,0,0.9927776336669922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,64,128,1,float16,float16,0,0.5527247905731201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,8,128,1,fp8,fp8,0,0.9020208358764649
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,8,128,1,float16,fp8,0,0.9622591972351074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,64,128,1,float16,fp8,0,0.5377520084381103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,64,128,1,fp8,fp8,0,0.5336688041687012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,1,128,1,float16,float16,0,0.4761504173278809
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,1,128,1,float16,fp8,0,0.467468786239624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,8,128,1,fp8,fp8,0,1.8102176666259766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,1,128,1,fp8,fp8,0,0.47986721992492676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,2,128,1,float16,float16,0,0.48987841606140137
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,2,128,1,float16,fp8,0,0.4536448001861572
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,2,128,1,fp8,fp8,0,0.4572688102722168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,4,128,1,float16,float16,0,0.48932638168334963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,4,128,1,float16,fp8,0,0.4694191932678223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,4,128,1,fp8,fp8,0,0.46512799263000487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,8,128,1,float16,float16,0,0.48058080673217773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,8,128,1,float16,fp8,0,0.45885281562805175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,64,8,128,1,fp8,fp8,0,0.4557983875274658
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,64,4,128,1,float16,float16,0,0.9723823547363282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,64,1,128,1,float16,float16,0,1.9480400085449219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,64,1,128,1,float16,fp8,0,8.508425903320312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,64,1,128,1,fp8,fp8,0,8.417269134521485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,64,2,128,1,float16,fp8,0,8.528849792480468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,64,2,128,1,fp8,fp8,0,8.435546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,64,1,128,1,float16,float16,0,10.120740509033203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,64,2,128,1,float16,float16,0,10.139579010009765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,64,4,128,1,float16,float16,0,9.912140655517579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,64,4,128,1,float16,fp8,0,8.4552001953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,64,128,1,float16,fp8,0,4.831423950195313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,64,128,1,float16,float16,0,5.7923744201660154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,1,128,1,float16,float16,0,4.600716781616211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,64,128,1,fp8,fp8,0,4.786308670043946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,64,4,128,1,fp8,fp8,0,8.755136108398437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,64,8,128,1,float16,fp8,0,8.694609832763671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,64,8,128,1,fp8,fp8,0,8.527598571777343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,64,8,128,1,float16,float16,0,10.232513427734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,1,128,1,float16,fp8,0,4.237891387939453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,1,128,1,fp8,fp8,0,4.2813232421875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,2,128,1,float16,fp8,0,4.220870590209961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,2,128,1,fp8,fp8,0,4.288888168334961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,2,128,1,float16,float16,0,4.907508850097656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,4,128,1,float16,fp8,0,4.226067352294922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,4,128,1,float16,float16,0,4.976766586303711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,4,128,1,fp8,fp8,0,4.319270324707031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,64,128,1,float16,fp8,0,2.4593759536743165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,8,128,1,float16,float16,0,5.109550476074219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,8,128,1,float16,fp8,0,4.303051376342774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,64,128,1,fp8,fp8,0,2.3584848403930665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,1,128,1,float16,float16,0,2.420798492431641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,1,128,1,float16,fp8,0,2.1106943130493163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,64,8,128,1,fp8,fp8,0,4.234931182861328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,1,128,1,fp8,fp8,0,2.1278656005859373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,2,128,1,float16,fp8,0,2.160436820983887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,2,128,1,float16,float16,0,2.4512367248535156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,2,128,1,fp8,fp8,0,2.1219615936279297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,4,128,1,float16,fp8,0,2.115255928039551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,4,128,1,float16,float16,0,2.4807247161865233
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,64,128,1,float16,float16,0,2.834556770324707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,64,128,1,float16,float16,0,1.3183152198791503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,64,128,1,float16,fp8,0,1.1860176086425782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,8,128,1,float16,float16,0,2.3752832412719727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,64,128,1,fp8,fp8,0,1.3121600151062012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,8,128,1,fp8,fp8,0,2.222515106201172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,1,128,1,float16,fp8,0,1.1465776443481446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,1,128,1,fp8,fp8,0,1.0630160331726075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,2,128,1,float16,fp8,0,1.107249641418457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,2,128,1,fp8,fp8,0,1.067572784423828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,4,128,1,float16,float16,0,1.1448736190795898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,4,128,1,float16,fp8,0,1.0670991897583009
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,4,128,1,fp8,fp8,0,2.125057601928711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,4,128,1,fp8,fp8,0,1.076859188079834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,8,128,1,float16,float16,0,1.148953628540039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,64,128,1,float16,float16,0,0.664572811126709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,64,128,1,float16,fp8,0,0.6075808048248291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,8,128,1,float16,fp8,0,1.1241600036621093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,64,128,1,fp8,fp8,0,0.6914095878601074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,64,8,128,1,float16,fp8,0,2.140614318847656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,1,128,1,float16,fp8,0,0.5391312122344971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,1,128,1,float16,float16,0,1.1086288452148438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,2,128,1,float16,float16,0,0.6157408237457276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,2,128,1,float16,fp8,0,0.5336880207061767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,2,128,1,fp8,fp8,0,0.5342351913452148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,4,128,1,float16,float16,0,0.5654784202575683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,2,128,1,float16,float16,0,1.226036834716797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,4,128,1,float16,fp8,0,0.5678175926208496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,4,128,1,fp8,fp8,0,0.5344336032867432
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,8,128,1,float16,float16,0,0.5742015838623047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,8,128,1,float16,fp8,0,0.5413072109222412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,64,128,1,float16,float16,0,0.34299681186676023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,8,128,1,fp8,fp8,0,0.5595471858978271
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,64,128,1,float16,fp8,0,0.3221136093139648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,1,128,1,float16,float16,0,0.28799359798431395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,1,128,1,float16,fp8,0,0.2798975944519043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,2,128,1,float16,float16,0,0.2895136117935181
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,2,128,1,float16,fp8,0,0.27665119171142577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,2,128,1,fp8,fp8,0,0.2730128049850464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,4,128,1,float16,float16,0,0.294321608543396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,64,8,128,1,fp8,fp8,0,1.073964786529541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,4,128,1,float16,fp8,0,0.2731472015380859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,4,128,1,fp8,fp8,0,0.27507519721984863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,1,128,1,float16,float16,0,0.564572811126709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,8,128,1,float16,float16,0,0.2881808042526245
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,64,1,128,1,fp8,fp8,0,0.553391981124878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,8,128,1,fp8,fp8,0,0.2757200002670288
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,64,128,1,fp8,fp8,0,0.3057744026184082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,1,128,1,fp8,fp8,0,0.2861232042312622
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,64,8,128,1,float16,fp8,0,0.27424159049987795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,64,1,128,1,float16,fp8,0,8.218342590332032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,64,1,128,1,fp8,fp8,0,8.264107513427735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,64,1,128,1,float16,float16,0,9.331263732910156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,64,2,128,1,float16,float16,0,9.693080139160156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,64,2,128,1,float16,fp8,0,8.214810943603515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,64,2,128,1,fp8,fp8,0,8.271520233154297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,64,4,128,1,float16,float16,0,9.589713287353515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,64,4,128,1,float16,fp8,0,8.234855651855469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,64,128,1,float16,fp8,0,4.782799911499024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,64,128,1,float16,float16,0,5.622219085693359
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,64,4,128,1,fp8,fp8,0,8.235639953613282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,64,8,128,1,float16,fp8,0,8.334129333496094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,64,8,128,1,float16,float16,0,9.666342163085938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,64,128,1,fp8,fp8,0,4.819460678100586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,64,8,128,1,fp8,fp8,0,8.392768096923827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,1,128,1,float16,float16,0,4.479537582397461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,1,128,1,float16,fp8,0,4.133841705322266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,1,128,1,fp8,fp8,0,4.220431900024414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,2,128,1,float16,float16,0,4.456391906738281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,2,128,1,float16,fp8,0,4.11401138305664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,2,128,1,fp8,fp8,0,4.170691299438476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,4,128,1,float16,fp8,0,4.150436782836914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,4,128,1,fp8,fp8,0,4.21629753112793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,4,128,1,float16,float16,0,4.7819969177246096
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,64,128,1,float16,float16,0,2.7271503448486327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,8,128,1,float16,float16,0,4.825361633300782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,64,128,1,float16,fp8,0,2.3899503707885743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,8,128,1,float16,fp8,0,4.161896133422852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,64,128,1,fp8,fp8,0,2.390822410583496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,1,128,1,float16,fp8,0,2.1623968124389648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,64,8,128,1,fp8,fp8,0,4.24183349609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,1,128,1,fp8,fp8,0,2.1446128845214845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,2,128,1,float16,fp8,0,2.0758432388305663
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,2,128,1,fp8,fp8,0,2.0666351318359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,4,128,1,float16,float16,0,2.2990320205688475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,4,128,1,fp8,fp8,0,2.080886459350586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,1,128,1,float16,float16,0,2.2179359436035155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,64,128,1,float16,fp8,0,1.2024016380310059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,8,128,1,float16,fp8,0,2.0988079071044923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,8,128,1,float16,float16,0,2.2985712051391602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,64,128,1,fp8,fp8,0,1.2098079681396485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,1,128,1,float16,float16,0,1.0869695663452148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,1,128,1,float16,fp8,0,1.0444255828857423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,1,128,1,fp8,fp8,0,1.1327343940734864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,2,128,1,float16,float16,0,2.323027229309082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,4,128,1,float16,fp8,0,2.0652448654174806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,2,128,1,float16,float16,0,1.0827152252197265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,2,128,1,float16,fp8,0,1.0791040420532227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,2,128,1,fp8,fp8,0,1.043563175201416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,4,128,1,float16,float16,0,1.1323871612548828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,4,128,1,float16,fp8,0,1.0417023658752442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,4,128,1,fp8,fp8,0,1.042527961730957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,8,128,1,float16,float16,0,1.1252047538757324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,8,128,1,float16,fp8,0,1.0635104179382324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,64,128,1,float16,float16,0,1.4177167892456055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,64,128,1,float16,float16,0,0.7274960041046142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,64,8,128,1,fp8,fp8,0,1.0558719635009766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,64,8,128,1,fp8,fp8,0,2.0911760330200195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,1,128,1,float16,float16,0,0.58155198097229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,1,128,1,float16,fp8,0,0.5286287784576416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,1,128,1,fp8,fp8,0,0.5261951923370362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,2,128,1,float16,float16,0,0.5792880058288574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,2,128,1,fp8,fp8,0,0.5252031803131103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,4,128,1,float16,float16,0,0.5716527938842774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,4,128,1,fp8,fp8,0,0.5269472122192382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,8,128,1,float16,float16,0,0.5740896224975586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,8,128,1,float16,fp8,0,0.5323232173919678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,64,128,1,float16,float16,0,0.3464992046356201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,8,128,1,fp8,fp8,0,0.5516928195953369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,64,128,1,float16,fp8,0,0.3199471950531006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,64,128,1,fp8,fp8,0,0.31334719657897947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,1,128,1,float16,float16,0,0.28076159954071045
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,1,128,1,fp8,fp8,0,0.27648320198059084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,2,128,1,float16,float16,0,0.28022398948669436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,2,128,1,float16,fp8,0,0.26897120475769043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,2,128,1,fp8,fp8,0,0.2729360103607178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,4,128,1,float16,float16,0,0.283080005645752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,4,128,1,float16,fp8,0,0.2702672004699707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,4,128,1,fp8,fp8,0,0.26975839138031005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,8,128,1,float16,float16,0,0.28809759616851804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,64,128,1,float16,fp8,0,0.607366418838501
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,8,128,1,fp8,fp8,0,0.2714735984802246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,64,128,1,fp8,fp8,0,0.6078720092773438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,64,128,1,float16,float16,0,0.18103359937667846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,64,128,1,float16,fp8,0,0.1630095958709717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,64,128,1,fp8,fp8,0,0.16286079883575438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,1,128,1,float16,float16,0,0.14654719829559326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,1,128,1,float16,fp8,0,0.13964799642562867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,1,128,1,fp8,fp8,0,0.1400704026222229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,2,128,1,float16,float16,0,0.14711999893188477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,2,128,1,float16,fp8,0,0.1399359941482544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,2,128,1,fp8,fp8,0,0.14083679914474487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,4,128,1,float16,float16,0,0.14820480346679688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,4,128,1,float16,fp8,0,0.14076000452041626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,4,128,1,fp8,fp8,0,0.14086560010910035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,8,128,1,float16,float16,0,0.15071200132369994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,8,128,1,float16,fp8,0,0.14141440391540527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,1,128,1,float16,fp8,0,0.273523211479187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,64,8,128,1,float16,fp8,0,0.2717664003372192
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,2,128,1,float16,fp8,0,0.5302815914154053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,64,4,128,1,float16,fp8,0,0.5288368225097656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,64,8,128,1,fp8,fp8,0,0.14196799993515014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,64,1,128,1,float16,fp8,0,5.065529632568359
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,64,1,128,1,float16,float16,0,5.49846076965332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,64,1,128,1,fp8,fp8,0,5.103091049194336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,64,2,128,1,float16,float16,0,5.565054321289063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,64,2,128,1,float16,fp8,0,5.0592399597167965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,64,2,128,1,fp8,fp8,0,5.067748641967773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,64,4,128,1,float16,float16,0,5.691630554199219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,64,4,128,1,float16,fp8,0,5.068603134155273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,64,4,128,1,fp8,fp8,0,5.092278289794922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,64,8,128,1,float16,fp8,0,5.148459243774414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,64,8,128,1,float16,float16,0,5.910124969482422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,64,128,1,float16,float16,0,3.460291290283203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,64,128,1,float16,fp8,0,3.0252511978149412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,64,128,1,fp8,fp8,0,3.0571584701538086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,64,8,128,1,fp8,fp8,0,5.172788619995117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,1,128,1,float16,fp8,0,2.545350456237793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,1,128,1,fp8,fp8,0,2.5461536407470704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,2,128,1,float16,float16,0,2.789027214050293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,2,128,1,fp8,fp8,0,2.538894462585449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,2,128,1,float16,fp8,0,2.6050575256347654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,4,128,1,float16,float16,0,2.820110321044922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,4,128,1,fp8,fp8,0,2.5537343978881837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,64,128,1,float16,float16,0,1.7049375534057618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,8,128,1,float16,float16,0,2.6751888275146483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,8,128,1,float16,fp8,0,2.6510143280029297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,8,128,1,fp8,fp8,0,2.572831916809082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,64,128,1,float16,fp8,0,1.6055408477783204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,64,128,1,fp8,fp8,0,1.5197327613830567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,1,128,1,float16,float16,0,1.361302375793457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,1,128,1,float16,fp8,0,1.3309856414794923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,1,128,1,fp8,fp8,0,1.276313591003418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,2,128,1,float16,float16,0,1.3079888343811035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,2,128,1,float16,fp8,0,1.2828224182128907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,2,128,1,fp8,fp8,0,1.2745183944702148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,4,128,1,float16,fp8,0,2.552351951599121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,4,128,1,float16,float16,0,1.3996800422668456
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,4,128,1,float16,fp8,0,1.2924160003662108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,4,128,1,fp8,fp8,0,1.2843952178955078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,8,128,1,float16,fp8,0,1.2900863647460938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,8,128,1,float16,float16,0,1.351796817779541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,64,1,128,1,float16,float16,0,2.6832559585571287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,64,8,128,1,fp8,fp8,0,1.2936271667480468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,64,128,1,float16,fp8,0,0.812656021118164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,64,128,1,fp8,fp8,0,0.7718624114990235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,1,128,1,float16,float16,0,0.6646815776824951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,1,128,1,fp8,fp8,0,0.7025839805603027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,2,128,1,float16,float16,0,0.6633920192718505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,2,128,1,float16,fp8,0,0.6447023868560791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,2,128,1,fp8,fp8,0,0.6566495895385742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,4,128,1,float16,float16,0,0.6704639911651611
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,4,128,1,float16,fp8,0,0.6574512004852295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,4,128,1,fp8,fp8,0,0.6719647884368897
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,8,128,1,float16,float16,0,0.6812655925750732
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,8,128,1,float16,fp8,0,0.6539040088653565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,8,128,1,fp8,fp8,0,0.6590640068054199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,64,128,1,float16,fp8,0,0.39767520427703856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,64,128,1,fp8,fp8,0,0.4002880096435547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,1,128,1,float16,fp8,0,0.3287008047103882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,1,128,1,fp8,fp8,0,0.33114399909973147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,2,128,1,float16,float16,0,0.33835198879241946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,2,128,1,float16,fp8,0,0.3342319965362549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,2,128,1,fp8,fp8,0,0.32898080348968506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,4,128,1,float16,float16,0,0.3422080039978027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,4,128,1,fp8,fp8,0,0.33240959644317625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,8,128,1,float16,float16,0,0.3488287925720215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,8,128,1,fp8,fp8,0,0.33422720432281494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,64,128,1,float16,float16,0,0.2257904052734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,64,128,1,float16,fp8,0,0.20521600246429444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,64,128,1,fp8,fp8,0,0.20448319911956786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,1,128,1,float16,float16,0,0.173852801322937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,1,128,1,float16,fp8,0,0.16978880167007446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,1,128,1,fp8,fp8,0,0.1706287980079651
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,2,128,1,float16,float16,0,0.17571840286254883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,2,128,1,float16,fp8,0,0.17158880233764648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,2,128,1,fp8,fp8,0,0.17047840356826782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,4,128,1,float16,float16,0,0.17719360589981079
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,4,128,1,float16,fp8,0,0.17210880517959595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,64,128,1,float16,float16,0,0.43186078071594236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,4,128,1,fp8,fp8,0,0.17220640182495117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,8,128,1,float16,float16,0,0.179750394821167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,1,128,1,float16,float16,0,0.34143519401550293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,8,128,1,fp8,fp8,0,0.17267520427703859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,64,128,1,float16,float16,0,0.11854399442672729
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,64,128,1,float16,fp8,0,0.11002559661865234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,64,128,1,fp8,fp8,0,0.11056640148162841
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,1,128,1,float16,float16,0,0.09300959706306458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,1,128,1,float16,fp8,0,0.08813920021057128
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,1,128,1,fp8,fp8,0,0.08773120045661927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,2,128,1,float16,float16,0,0.0937391996383667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,2,128,1,float16,fp8,0,0.08798720240592957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,8,128,1,float16,fp8,0,0.33478560447692873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,2,128,1,fp8,fp8,0,0.08837440013885497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,4,128,1,float16,float16,0,0.09473440051078796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,4,128,1,float16,fp8,0,0.08907999992370605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,4,128,1,fp8,fp8,0,0.08895040154457093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,64,128,1,float16,float16,0,0.8467951774597168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,8,128,1,float16,fp8,0,0.09119840264320374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,64,1,128,1,float16,fp8,0,0.6426032066345215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,64,8,128,1,float16,fp8,0,0.17381759881973266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,64,4,128,1,float16,fp8,0,0.33019840717315674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,8,128,1,float16,float16,0,0.09621599912643433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,64,8,128,1,fp8,fp8,0,0.09034720063209534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,64,1,128,1,float16,fp8,0,5.314191818237305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,64,1,128,1,fp8,fp8,0,5.273942565917968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,64,2,128,1,float16,float16,0,5.578152084350586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,64,2,128,1,float16,fp8,0,5.294894409179688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,64,2,128,1,fp8,fp8,0,5.287332916259766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,64,1,128,1,float16,float16,0,5.543993759155273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,64,4,128,1,float16,float16,0,5.729163360595703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,64,4,128,1,float16,fp8,0,5.3211006164550785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,64,4,128,1,fp8,fp8,0,5.27882080078125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,64,128,1,float16,float16,0,3.7387134552001955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,64,128,1,fp8,fp8,0,3.304118347167969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,64,8,128,1,float16,float16,0,5.829934310913086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,1,128,1,float16,float16,0,2.6548959732055666
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,1,128,1,float16,fp8,0,2.6400304794311524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,64,8,128,1,fp8,fp8,0,5.386767959594726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,1,128,1,fp8,fp8,0,2.702017593383789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,2,128,1,float16,float16,0,2.7805759429931642
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,2,128,1,float16,fp8,0,2.7226415634155274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,2,128,1,fp8,fp8,0,2.693289566040039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,64,128,1,float16,fp8,0,3.363963317871094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,64,8,128,1,float16,fp8,0,5.373004913330078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,4,128,1,float16,float16,0,2.799660873413086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,64,128,1,float16,float16,0,1.8416208267211913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,4,128,1,fp8,fp8,0,2.6834432601928713
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,8,128,1,float16,float16,0,2.9710927963256837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,64,128,1,fp8,fp8,0,1.7402751922607422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,8,128,1,fp8,fp8,0,2.716035270690918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,1,128,1,float16,float16,0,1.3808624267578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,1,128,1,float16,fp8,0,1.3404959678649901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,1,128,1,fp8,fp8,0,1.4400992393493652
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,2,128,1,float16,float16,0,1.3723471641540528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,4,128,1,float16,fp8,0,2.670199966430664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,2,128,1,float16,fp8,0,1.3405360221862792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,4,128,1,float16,float16,0,1.3687199592590331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,4,128,1,float16,fp8,0,1.3533424377441405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,4,128,1,fp8,fp8,0,1.3814687728881836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,8,128,1,float16,float16,0,1.395852756500244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,8,128,1,float16,fp8,0,1.368124771118164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,64,128,1,float16,fp8,0,1.6604799270629882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,64,8,128,1,float16,fp8,0,2.721990394592285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,8,128,1,fp8,fp8,0,1.3926048278808594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,1,128,1,float16,float16,0,0.678865623474121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,64,128,1,float16,fp8,0,0.8402704238891602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,64,128,1,fp8,fp8,0,0.8595392227172851
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,1,128,1,float16,fp8,0,0.6722447872161865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,1,128,1,fp8,fp8,0,0.6775008201599121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,2,128,1,float16,float16,0,0.6838335990905762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,2,128,1,float16,fp8,0,0.6775887966156006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,2,128,1,fp8,fp8,0,0.671236801147461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,4,128,1,float16,float16,0,0.7018640041351318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,4,128,1,float16,fp8,0,0.6749472141265869
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,4,128,1,fp8,fp8,0,0.6806511878967285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,64,128,1,float16,float16,0,0.4736976146697998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,8,128,1,float16,float16,0,0.7062911987304688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,8,128,1,fp8,fp8,0,0.6843472003936768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,64,128,1,fp8,fp8,0,0.42897281646728513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,64,2,128,1,fp8,fp8,0,1.3717071533203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,1,128,1,float16,fp8,0,0.34464321136474607
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,1,128,1,fp8,fp8,0,0.3416095972061157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,2,128,1,float16,float16,0,0.3474208116531372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,2,128,1,float16,fp8,0,0.34324638843536376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,2,128,1,fp8,fp8,0,0.3443504095077515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,4,128,1,float16,float16,0,0.34916799068450927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,4,128,1,float16,fp8,0,0.3471152067184448
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,4,128,1,fp8,fp8,0,0.3451807975769043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,8,128,1,float16,float16,0,0.360152006149292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,8,128,1,float16,fp8,0,0.3484816074371338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,8,128,1,fp8,fp8,0,0.3494159936904907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,64,128,1,float16,fp8,0,0.2212928056716919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,64,128,1,fp8,fp8,0,0.21962718963623046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,64,128,1,float16,float16,0,0.9332943916320801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,1,128,1,float16,float16,0,0.17858400344848632
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,1,128,1,float16,fp8,0,0.17792160511016847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,1,128,1,fp8,fp8,0,0.17633119821548462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,2,128,1,float16,fp8,0,0.17752480506896973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,2,128,1,float16,float16,0,0.17899359464645387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,2,128,1,fp8,fp8,0,0.17685439586639404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,4,128,1,float16,float16,0,0.18096159696578978
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,4,128,1,float16,fp8,0,0.17896159887313842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,8,128,1,float16,float16,0,0.18619999885559083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,8,128,1,float16,fp8,0,0.18113919496536254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,8,128,1,fp8,fp8,0,0.18025120496749877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,64,128,1,float16,float16,0,0.1310703992843628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,64,128,1,float16,fp8,0,0.11725120544433594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,64,128,1,fp8,fp8,0,0.11725280284881592
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,64,128,1,float16,fp8,0,0.4247312068939209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,1,128,1,float16,fp8,0,0.09439679980278015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,1,128,1,fp8,fp8,0,0.09405120015144348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,2,128,1,float16,float16,0,0.09656640291213989
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,2,128,1,float16,fp8,0,0.09425600171089173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,64,1,128,1,float16,float16,0,0.3452944040298462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,4,128,1,float16,float16,0,0.09734560251235962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,4,128,1,float16,fp8,0,0.09551519751548768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,4,128,1,fp8,fp8,0,0.09533119797706605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,8,128,1,float16,float16,0,0.09950079917907714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,8,128,1,float16,fp8,0,0.09593279957771302
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,64,128,1,float16,float16,0,0.07012959718704223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,64,128,1,float16,fp8,0,0.0632863998413086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,64,128,1,fp8,fp8,0,0.06291360259056092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,1,128,1,float16,float16,0,0.049348801374435425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,64,128,1,float16,float16,0,0.2449183940887451
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,1,128,1,float16,fp8,0,0.04950880110263824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,1,128,1,fp8,fp8,0,0.049675199389457705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,2,128,1,float16,float16,0,0.04984799921512604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,2,128,1,float16,fp8,0,0.04973919987678528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,2,128,1,fp8,fp8,0,0.04960800111293793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,4,128,1,float16,float16,0,0.05095040202140808
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,4,128,1,float16,fp8,0,0.04947839975357056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,4,128,1,fp8,fp8,0,0.04984480142593384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,8,128,1,float16,float16,0,0.05282880067825317
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,8,128,1,float16,fp8,0,0.050406402349472045
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,64,4,128,1,fp8,fp8,0,0.17762240171432495
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,1,128,1,float16,float16,0,0.09565759897232055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,64,8,128,1,float16,fp8,0,0.6904064178466797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,2,128,1,fp8,fp8,0,0.09447199702262879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,64,8,128,1,fp8,fp8,0,0.09625279903411865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,64,8,128,1,fp8,fp8,0,0.04998719990253449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,64,1,128,1,float16,float16,0,3.7969009399414064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,64,1,128,1,float16,fp8,0,3.832112121582031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,64,2,128,1,float16,float16,0,3.765377426147461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,64,1,128,1,fp8,fp8,0,3.8472991943359376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,64,2,128,1,float16,fp8,0,3.8130878448486327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,64,2,128,1,fp8,fp8,0,3.8340816497802734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,64,4,128,1,float16,float16,0,3.881480026245117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,64,4,128,1,float16,fp8,0,3.8679359436035154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,64,4,128,1,fp8,fp8,0,3.909515380859375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,64,8,128,1,float16,float16,0,4.054051208496094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,64,128,1,float16,float16,0,2.865127944946289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,64,128,1,float16,fp8,0,2.6184623718261717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,64,8,128,1,float16,fp8,0,3.969289779663086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,64,8,128,1,fp8,fp8,0,3.9494400024414062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,1,128,1,float16,float16,0,1.8804927825927735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,64,128,1,fp8,fp8,0,2.63677921295166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,1,128,1,float16,fp8,0,1.9434223175048828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,1,128,1,fp8,fp8,0,1.9853727340698242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,2,128,1,float16,float16,0,1.8857952117919923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,2,128,1,float16,fp8,0,1.9257999420166017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,4,128,1,float16,float16,0,1.9255199432373047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,4,128,1,float16,fp8,0,1.943231964111328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,4,128,1,fp8,fp8,0,1.9586784362792968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,8,128,1,float16,float16,0,2.003108787536621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,8,128,1,float16,fp8,0,1.9780096054077148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,64,128,1,float16,float16,0,1.4470784187316894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,64,128,1,float16,fp8,0,1.3176591873168946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,1,128,1,float16,float16,0,0.9530976295471192
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,64,128,1,fp8,fp8,0,1.3176063537597655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,1,128,1,float16,fp8,0,0.992033576965332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,1,128,1,fp8,fp8,0,0.9698512077331543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,2,128,1,float16,float16,0,0.9464847564697265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,2,128,1,float16,fp8,0,0.9707311630249024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,2,128,1,fp8,fp8,0,0.9798303604125976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,4,128,1,float16,float16,0,0.9708512306213379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,4,128,1,float16,fp8,0,0.9822175979614258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,2,128,1,fp8,fp8,0,1.9398479461669922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,8,128,1,float16,float16,0,0.9979519844055176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,8,128,1,float16,fp8,0,0.995963191986084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,64,128,1,float16,float16,0,0.7342480182647705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,8,128,1,fp8,fp8,0,1.0026512145996094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,64,128,1,float16,fp8,0,0.6693600177764892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,64,128,1,fp8,fp8,0,0.6720655918121338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,1,128,1,float16,fp8,0,0.4920656204223633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,2,128,1,float16,float16,0,0.4837376117706299
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,1,128,1,fp8,fp8,0,0.4938240051269531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,64,8,128,1,fp8,fp8,0,1.9829088211059571
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,2,128,1,fp8,fp8,0,0.49602880477905276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,4,128,1,float16,float16,0,0.49101920127868653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,4,128,1,float16,fp8,0,0.49750399589538574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,4,128,1,fp8,fp8,0,0.4984591960906982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,8,128,1,float16,float16,0,0.5102399826049805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,8,128,1,fp8,fp8,0,0.5058256149291992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,64,128,1,float16,float16,0,0.3746239900588989
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,64,128,1,float16,fp8,0,0.3416143894195557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,64,128,1,fp8,fp8,0,0.34202721118927004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,1,128,1,float16,fp8,0,0.2522255897521973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,2,128,1,float16,float16,0,0.24468319416046141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,2,128,1,float16,fp8,0,0.24904799461364746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,2,128,1,fp8,fp8,0,0.25007200241088867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,4,128,1,float16,float16,0,0.25079519748687745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,1,128,1,float16,float16,0,0.47768001556396483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,4,128,1,float16,fp8,0,0.25148639678955076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,4,128,1,fp8,fp8,0,0.2523200035095215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,2,128,1,float16,fp8,0,0.49309601783752444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,8,128,1,float16,float16,0,0.25842719078063964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,8,128,1,float16,fp8,0,0.25873920917510984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,64,128,1,float16,float16,0,0.1942479968070984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,64,128,1,float16,fp8,0,0.17648160457611084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,64,128,1,fp8,fp8,0,0.17553759813308717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,64,8,128,1,float16,fp8,0,0.5016943931579589
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,1,128,1,float16,float16,0,0.12902079820632933
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,1,128,1,float16,fp8,0,0.1302175998687744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,1,128,1,fp8,fp8,0,0.12973439693450928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,2,128,1,float16,float16,0,0.13006240129470825
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,2,128,1,float16,fp8,0,0.13061280250549318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,2,128,1,fp8,fp8,0,0.13019360303878785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,1,128,1,fp8,fp8,0,0.2518176078796387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,4,128,1,float16,float16,0,0.13179199695587157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,4,128,1,float16,fp8,0,0.13182879686355592
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,4,128,1,fp8,fp8,0,0.13132959604263306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,8,128,1,float16,float16,0,0.13548959493637086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,8,128,1,fp8,fp8,0,0.1337007999420166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,64,128,1,float16,float16,0,0.10404800176620484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,64,128,1,float16,fp8,0,0.09486560225486755
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,64,128,1,fp8,fp8,0,0.09484000205993652
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,1,128,1,float16,float16,0,0.06937440037727356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,1,128,1,float16,fp8,0,0.07069600224494935
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,64,4,128,1,fp8,fp8,0,0.974830436706543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,2,128,1,float16,float16,0,0.06984000205993653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,2,128,1,float16,fp8,0,0.07090719938278198
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,4,128,1,float16,float16,0,0.07116000056266784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,2,128,1,fp8,fp8,0,0.07190399765968322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,4,128,1,fp8,fp8,0,0.0719327986240387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,8,128,1,float16,float16,0,0.07376959919929504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,8,128,1,fp8,fp8,0,0.07279360294342041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,64,128,1,float16,fp8,0,0.053878402709960936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,64,128,1,fp8,fp8,0,0.05415679812431336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,1,128,1,float16,float16,0,0.03678080141544342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,1,128,1,float16,fp8,0,0.03800640106201172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,1,128,1,fp8,fp8,0,0.03806560039520264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,2,128,1,float16,float16,0,0.03690719902515412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,2,128,1,float16,fp8,0,0.03808160126209259
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,8,128,1,fp8,fp8,0,0.25963521003723145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,2,128,1,fp8,fp8,0,0.03823840022087097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,4,128,1,float16,float16,0,0.03784320056438446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,4,128,1,float16,fp8,0,0.038176000118255615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,4,128,1,fp8,fp8,0,0.03838239908218384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,8,128,1,float16,float16,0,0.03952319920063019
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,8,128,1,float16,fp8,0,0.038780799508094786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,8,128,1,fp8,fp8,0,0.038396799564361574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,64,128,1,float16,float16,0,0.03611040115356445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,64,128,1,float16,fp8,0,0.03091840147972107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,64,1,128,1,float16,float16,0,0.24579520225524903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,1,128,1,float16,float16,0,0.02252320051193237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,1,128,1,float16,fp8,0,0.024108800292015075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,1,128,1,fp8,fp8,0,0.02421119958162308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,2,128,1,float16,float16,0,0.02266400009393692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,2,128,1,float16,fp8,0,0.024142399430274963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,2,128,1,fp8,fp8,0,0.02414879947900772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,64,8,128,1,float16,fp8,0,0.13435200452804566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,4,128,1,float16,fp8,0,0.024081599712371827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,4,128,1,fp8,fp8,0,0.024302400648593903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,8,128,1,float16,float16,0,0.022972799837589264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,8,128,1,float16,fp8,0,0.024139200150966645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,8,128,1,fp8,fp8,0,0.02438720017671585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,1,128,1,fp8,fp8,0,0.07076960206031799
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,4,128,1,float16,fp8,0,0.07191039919853211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,64,8,128,1,float16,fp8,0,0.07265920042991639
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,64,1,128,1,float16,float16,0,1.480179214477539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,64,1,128,1,float16,fp8,0,1.5747376441955567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,64,1,128,1,fp8,fp8,0,1.576318359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,64,2,128,1,float16,float16,0,1.4838000297546388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,64,2,128,1,float16,fp8,0,1.562342357635498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,64,128,1,fp8,fp8,0,0.030588799715042116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,64,4,128,1,float16,float16,0,0.02282080054283142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,64,2,128,1,fp8,fp8,0,1.5746368408203124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,64,4,128,1,float16,float16,0,1.5213888168334961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,64,4,128,1,float16,fp8,0,1.5873295783996582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,64,4,128,1,fp8,fp8,0,1.576480007171631
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,64,64,128,1,float16,float16,0,0.05859360098838806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,64,8,128,1,float16,float16,0,1.582806396484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,64,8,128,1,float16,fp8,0,1.6192960739135742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,64,128,1,float16,fp8,0,1.0933103561401367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,64,128,1,float16,float16,0,1.268777561187744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,64,8,128,1,fp8,fp8,0,1.6118560791015626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,1,128,1,float16,fp8,0,0.7887519836425781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,1,128,1,fp8,fp8,0,0.7899487972259521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,2,128,1,float16,float16,0,0.7516032218933105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,2,128,1,float16,fp8,0,0.7925600051879883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,2,128,1,fp8,fp8,0,0.7922976016998291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,4,128,1,float16,float16,0,0.7661695957183838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,4,128,1,float16,fp8,0,0.7956495761871338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,4,128,1,fp8,fp8,0,0.7955760002136231
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,8,128,1,float16,float16,0,0.8019040107727051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,8,128,1,float16,fp8,0,0.8159248352050781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,64,128,1,float16,float16,0,0.6427472114562989
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,8,128,1,fp8,fp8,0,0.8162351608276367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,64,128,1,float16,fp8,0,0.5520016193389893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,64,128,1,fp8,fp8,0,0.5522272109985351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,1,128,1,float16,fp8,0,0.40083842277526854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,1,128,1,fp8,fp8,0,0.4027696132659912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,2,128,1,float16,float16,0,0.381276798248291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,2,128,1,float16,fp8,0,0.40006561279296876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,2,128,1,fp8,fp8,0,0.3995327949523926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,4,128,1,float16,float16,0,0.3915535926818848
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,4,128,1,float16,fp8,0,0.40476160049438475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,1,128,1,float16,float16,0,0.7427536010742187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,64,64,128,1,fp8,fp8,0,1.0951503753662108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,8,128,1,float16,float16,0,0.4048480033874512
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,8,128,1,float16,fp8,0,0.4148496150970459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,8,128,1,fp8,fp8,0,0.4129007816314697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,64,128,1,float16,float16,0,0.329256010055542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,1,128,1,float16,float16,0,0.19681119918823242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,64,128,1,fp8,fp8,0,0.28333120346069335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,64,128,1,float16,fp8,0,0.2844912052154541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,1,128,1,fp8,fp8,0,0.20647358894348145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,2,128,1,float16,fp8,0,0.20559520721435548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,2,128,1,fp8,fp8,0,0.20726559162139893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,4,128,1,float16,float16,0,0.2008431911468506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,4,128,1,float16,fp8,0,0.20743680000305176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,4,128,1,fp8,fp8,0,0.2092128038406372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,8,128,1,float16,float16,0,0.20923359394073487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,8,128,1,float16,fp8,0,0.21242880821228027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,8,128,1,fp8,fp8,0,0.21349918842315674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,64,128,1,float16,float16,0,0.1719920039176941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,64,128,1,float16,fp8,0,0.14835360050201415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,64,128,1,fp8,fp8,0,0.1488752007484436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,1,128,1,float16,float16,0,0.10412479639053344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,1,128,1,float16,fp8,0,0.10820480585098266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,1,128,1,float16,float16,0,0.3817568063735962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,2,128,1,float16,float16,0,0.10498239994049072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,2,128,1,float16,fp8,0,0.10858559608459473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,2,128,1,fp8,fp8,0,0.10941280126571655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,4,128,1,float16,float16,0,0.10697760581970214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,4,128,1,float16,fp8,0,0.1101040005683899
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,4,128,1,fp8,fp8,0,0.11034239530563354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,8,128,1,float16,float16,0,0.11146559715270996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,8,128,1,float16,fp8,0,0.11212320327758789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,8,128,1,fp8,fp8,0,0.11271519660949707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,64,128,1,float16,float16,0,0.0938431978225708
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,64,128,1,float16,fp8,0,0.08151360154151917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,64,128,1,fp8,fp8,0,0.08174719810485839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,1,128,1,float16,fp8,0,0.06027519702911377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,64,4,128,1,fp8,fp8,0,0.40375680923461915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,1,128,1,fp8,fp8,0,0.06029760241508484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,2,128,1,float16,float16,0,0.05893440246582031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,2,128,1,fp8,fp8,0,0.06032000184059143
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,4,128,1,float16,float16,0,0.05999040007591248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,4,128,1,float16,fp8,0,0.06100000143051147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,4,128,1,fp8,fp8,0,0.060838401317596436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,8,128,1,float16,float16,0,0.06229119896888733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,8,128,1,float16,fp8,0,0.06190400123596192
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,8,128,1,fp8,fp8,0,0.062196797132492064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,64,128,1,float16,float16,0,0.05279039740562439
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,64,128,1,float16,fp8,0,0.04672479927539826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,64,128,1,fp8,fp8,0,0.04629440009593964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,2,128,1,float16,float16,0,0.19805599451065065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,1,128,1,fp8,fp8,0,0.032488000392913816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,2,128,1,float16,float16,0,0.030958399176597595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,2,128,1,float16,fp8,0,0.03255999982357025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,2,128,1,fp8,fp8,0,0.03262239992618561
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,4,128,1,float16,float16,0,0.0319568008184433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,4,128,1,float16,fp8,0,0.03261919915676117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,4,128,1,fp8,fp8,0,0.03259679973125458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,8,128,1,float16,float16,0,0.03418720066547394
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,64,1,128,1,fp8,fp8,0,0.10901919603347779
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,64,128,1,float16,float16,0,0.032199999690055846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,8,128,1,fp8,fp8,0,0.03365600109100342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,64,128,1,float16,fp8,0,0.025374400615692138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,64,128,1,fp8,fp8,0,0.025196799635887147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,1,128,1,float16,float16,0,0.019225600361824035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,1,128,1,float16,fp8,0,0.020336000621318816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,1,128,1,fp8,fp8,0,0.020529599487781526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,2,128,1,float16,float16,0,0.019392000138759614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,2,128,1,float16,fp8,0,0.02056639939546585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,2,128,1,fp8,fp8,0,0.020396800339221956
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,4,128,1,float16,float16,0,0.019676800072193145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,4,128,1,float16,fp8,0,0.02051520049571991
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,4,128,1,fp8,fp8,0,0.020454399287700653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,1,128,1,float16,float16,0,0.05786240100860596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,8,128,1,float16,float16,0,0.019998399913311003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,8,128,1,float16,fp8,0,0.020604799687862396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,64,8,128,1,fp8,fp8,0,0.0206496000289917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,64,128,1,float16,fp8,0,0.016774399578571318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,64,128,1,fp8,fp8,0,0.016833600401878358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,64,2,128,1,float16,fp8,0,0.0601360023021698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,1,128,1,float16,fp8,0,0.015140800178050995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,1,128,1,fp8,fp8,0,0.015452800691127777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,2,128,1,float16,float16,0,0.014640000462532044
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,2,128,1,float16,fp8,0,0.015329599380493164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,2,128,1,fp8,fp8,0,0.015057599544525147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,4,128,1,float16,float16,0,0.014905600249767304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,4,128,1,float16,fp8,0,0.01510400027036667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,4,128,1,fp8,fp8,0,0.015193599462509155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,8,128,1,float16,float16,0,0.01488800048828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,64,1,128,1,float16,fp8,0,0.20660479068756105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,8,128,1,float16,fp8,0,0.015304000675678253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,1,128,1,float16,float16,0,0.03091840147972107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,8,128,1,fp8,fp8,0,0.015276800096035003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,1,128,1,float16,fp8,0,0.03249120116233826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,64,1,128,1,float16,float16,0,0.681279993057251
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,64,1,128,1,float16,fp8,0,0.7454927921295166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,64,2,128,1,float16,float16,0,0.6819104194641114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,64,8,128,1,float16,fp8,0,0.03283520042896271
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,64,2,128,1,float16,fp8,0,0.7398464202880859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,64,2,128,1,fp8,fp8,0,0.7464911937713623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,64,4,128,1,float16,fp8,0,0.7553423881530762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,64,4,128,1,fp8,fp8,0,0.7497392177581788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,64,8,128,1,float16,float16,0,0.7286719799041748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,64,8,128,1,float16,fp8,0,0.7788623809814453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,64,8,128,1,fp8,fp8,0,0.7743103981018067
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,64,128,1,float16,fp8,0,0.549124813079834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,64,128,1,fp8,fp8,0,0.5481103897094727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,64,1,128,1,fp8,fp8,0,0.7368991851806641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,1,128,1,float16,float16,0,0.3502847909927368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,1,128,1,float16,fp8,0,0.3742703914642334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,1,128,1,fp8,fp8,0,0.37435359954833985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,2,128,1,float16,fp8,0,0.374456000328064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,2,128,1,fp8,fp8,0,0.3740112066268921
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,4,128,1,float16,float16,0,0.3565711975097656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,4,128,1,float16,fp8,0,0.3786688089370728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,1,128,1,float16,float16,0,0.01462399959564209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,4,128,1,fp8,fp8,0,0.37851839065551757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,8,128,1,float16,float16,0,0.3688096046447754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,8,128,1,float16,fp8,0,0.39599359035491943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,8,128,1,fp8,fp8,0,0.3930511951446533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,64,128,1,float16,float16,0,0.6073040008544922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,64,128,1,float16,fp8,0,0.2813136100769043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,64,128,1,fp8,fp8,0,0.2807919979095459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,1,128,1,float16,float16,0,0.17831840515136718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,1,128,1,float16,fp8,0,0.19291360378265382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,1,128,1,fp8,fp8,0,0.1907968044281006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,2,128,1,float16,float16,0,0.18245919942855834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,2,128,1,float16,fp8,0,0.1946336030960083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,2,128,1,fp8,fp8,0,0.19202400445938111
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,4,128,1,float16,float16,0,0.18482719659805297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,4,128,1,float16,fp8,0,0.19585599899291992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,4,128,1,fp8,fp8,0,0.19364800453186035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,8,128,1,float16,float16,0,0.19253920316696166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,64,2,128,1,float16,float16,0,0.3505359888076782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,8,128,1,float16,fp8,0,0.2039504051208496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,8,128,1,fp8,fp8,0,0.2003024101257324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,1,128,1,float16,float16,0,0.09549919962882995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,64,128,1,float16,fp8,0,0.1472767949104309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,64,128,1,fp8,fp8,0,0.14650239944458007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,1,128,1,float16,fp8,0,0.1033776044845581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,1,128,1,fp8,fp8,0,0.10389920473098754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,2,128,1,float16,fp8,0,0.10408159494400024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,4,128,1,float16,float16,0,0.10011039972305298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,4,128,1,float16,fp8,0,0.10513440370559693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,4,128,1,fp8,fp8,0,0.10609279870986939
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,8,128,1,float16,float16,0,0.10260000228881835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,8,128,1,float16,fp8,0,0.10824639797210693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,8,128,1,fp8,fp8,0,0.10799520015716553
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,64,128,1,float16,float16,0,0.08939520120620728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,64,128,1,float16,fp8,0,0.0780672013759613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,64,128,1,fp8,fp8,0,0.07866079807281494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,1,128,1,float16,float16,0,0.05426560044288635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,1,128,1,float16,fp8,0,0.05642719864845276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,1,128,1,fp8,fp8,0,0.056199997663497925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,2,128,1,float16,float16,0,0.05437440276145935
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,2,128,1,float16,fp8,0,0.05645279884338379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,64,64,128,1,float16,float16,0,0.3126303911209106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,4,128,1,float16,float16,0,0.05541279911994934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,4,128,1,float16,fp8,0,0.05728800296783447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,4,128,1,fp8,fp8,0,0.05713599920272827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,8,128,1,float16,float16,0,0.058171200752258304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,8,128,1,float16,fp8,0,0.05856000185012818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,8,128,1,fp8,fp8,0,0.05853279829025269
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,64,128,1,float16,float16,0,0.05029439926147461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,64,128,1,fp8,fp8,0,0.04273439943790436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,1,128,1,float16,float16,0,0.02747200131416321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,1,128,1,float16,fp8,0,0.029135999083518983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,1,128,1,fp8,fp8,0,0.02911199927330017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,2,128,1,float16,float16,0,0.027953600883483885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,2,128,1,float16,fp8,0,0.029334399104118346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,64,128,1,float16,float16,0,0.1651039958000183
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,64,4,128,1,float16,float16,0,0.698956823348999
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,2,128,1,float16,float16,0,0.09674879908561707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,4,128,1,float16,fp8,0,0.02922559976577759
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,4,128,1,fp8,fp8,0,0.029319998621940613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,8,128,1,float16,float16,0,0.02975200116634369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,8,128,1,float16,fp8,0,0.02967199981212616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,8,128,1,fp8,fp8,0,0.029734399914741517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,64,128,1,float16,float16,0,0.028756800293922424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,64,128,1,float16,fp8,0,0.021161599457263945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,64,64,128,1,float16,float16,0,0.017254400253295898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,1,128,1,float16,float16,0,0.017668800055980684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,1,128,1,fp8,fp8,0,0.0188511997461319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,1,128,1,float16,fp8,0,0.018780800700187682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,2,128,1,float16,float16,0,0.017740799486637114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,2,128,1,float16,fp8,0,0.01860000044107437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,2,128,1,fp8,fp8,0,0.018718400597572328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,4,128,1,float16,float16,0,0.017811200022697447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,4,128,1,fp8,fp8,0,0.01865279972553253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,64,2,128,1,fp8,fp8,0,0.05648159980773926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,8,128,1,float16,float16,0,0.018246400356292724
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,8,128,1,float16,fp8,0,0.018772800266742707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,8,128,1,fp8,fp8,0,0.018799999356269838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,64,128,1,float16,float16,0,0.015355199575424194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,64,128,1,float16,fp8,0,0.042027199268341066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,64,128,1,fp8,fp8,0,0.014878399670124054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,1,128,1,float16,float16,0,0.012907199561595917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,1,128,1,float16,fp8,0,0.013604800403118133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,1,128,1,fp8,fp8,0,0.013652800023555756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,2,128,1,float16,float16,0,0.013076800107955932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,2,128,1,float16,fp8,0,0.013513599336147309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,2,128,1,fp8,fp8,0,0.02940639853477478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,4,128,1,float16,float16,0,0.013131199777126313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,4,128,1,float16,fp8,0,0.013692800700664521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,4,128,1,fp8,fp8,0,0.013558399677276612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,64,2,128,1,fp8,fp8,0,0.10337920188903808
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,8,128,1,float16,float16,0,0.013179199397563934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,8,128,1,float16,fp8,0,0.013686400651931763
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,8,128,1,fp8,fp8,0,0.013961599767208099
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,64,128,1,float16,float16,0,0.014511999487876893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,64,128,1,float16,fp8,0,0.013984000682830811
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,64,128,1,fp8,fp8,0,0.014203199744224548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,1,128,1,float16,float16,0,0.012544000148773193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,1,128,1,float16,fp8,0,0.013313600420951843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,1,128,1,fp8,fp8,0,0.01319040060043335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,2,128,1,float16,float16,0,0.012561599910259246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,2,128,1,float16,fp8,0,0.013299199938774108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,2,128,1,fp8,fp8,0,0.013121600449085235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,4,128,1,float16,float16,0,0.012756800651550293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,4,128,1,float16,fp8,0,0.013150399923324585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,4,128,1,fp8,fp8,0,0.013395200669765472
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,4,128,1,float16,fp8,0,0.018646399676799773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,8,128,1,float16,fp8,0,0.01348000019788742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,8,128,1,fp8,fp8,0,0.013143999874591828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,64,1,128,1,float16,float16,0,0.34833440780639646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,64,1,128,1,float16,fp8,0,0.3759471893310547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,64,1,128,1,fp8,fp8,0,0.3743936061859131
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,64,128,1,float16,fp8,0,0.014864000678062438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,64,2,128,1,float16,float16,0,0.3498176097869873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,64,2,128,1,float16,fp8,0,0.37512640953063964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,64,4,128,1,float16,float16,0,0.3580575942993164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,64,2,128,1,fp8,fp8,0,0.013518400490283966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,64,4,128,1,float16,fp8,0,0.3784127950668335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,64,4,128,1,fp8,fp8,0,0.3795599937438965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,64,8,128,1,float16,float16,0,0.37092640399932864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,64,8,128,1,float16,fp8,0,0.39206719398498535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,64,8,128,1,fp8,fp8,0,0.3941359996795654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,64,64,128,1,fp8,fp8,0,0.021614399552345277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,64,128,1,float16,float16,0,0.4277071952819824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,64,128,1,float16,fp8,0,0.4153151988983154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,64,128,1,fp8,fp8,0,0.4169136047363281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,64,8,128,1,float16,float16,0,0.012780800461769104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,2,128,1,float16,float16,0,0.17833919525146485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,2,128,1,float16,fp8,0,0.1923311948776245
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,2,128,1,fp8,fp8,0,0.19046880006790162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,4,128,1,float16,float16,0,0.1836303949356079
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,4,128,1,float16,fp8,0,0.19474719762802123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,4,128,1,fp8,fp8,0,0.192631995677948
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,8,128,1,float16,float16,0,0.193995201587677
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,8,128,1,float16,fp8,0,0.20143840312957764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,8,128,1,fp8,fp8,0,0.20023040771484374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,64,128,1,float16,float16,0,0.2212048053741455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,64,128,1,float16,fp8,0,0.2163248062133789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,1,128,1,float16,float16,0,0.09530559778213502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,64,128,1,fp8,fp8,0,0.21589438915252684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,1,128,1,float16,fp8,0,0.10380480289459229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,1,128,1,fp8,fp8,0,0.10408960580825806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,2,128,1,float16,float16,0,0.09626240134239197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,64,2,128,1,fp8,fp8,0,0.37779200077056885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,2,128,1,fp8,fp8,0,0.10447200536727905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,4,128,1,float16,float16,0,0.09876800179481507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,4,128,1,float16,fp8,0,0.10603200197219849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,4,128,1,fp8,fp8,0,0.105622398853302
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,8,128,1,float16,float16,0,0.1027184009552002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,8,128,1,float16,fp8,0,0.10874400138854981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,64,4,128,1,float16,float16,0,0.02831200063228607
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,8,128,1,fp8,fp8,0,0.10821759700775146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,64,128,1,float16,float16,0,0.11859040260314942
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,64,128,1,float16,fp8,0,0.11405119895935059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,1,128,1,float16,float16,0,0.05414400100708008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,1,128,1,float16,fp8,0,0.05653280019760132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,1,128,1,fp8,fp8,0,0.05686560273170471
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,2,128,1,float16,fp8,0,0.057251197099685666
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,2,128,1,fp8,fp8,0,0.057145601511001586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,4,128,1,float16,float16,0,0.05659040212631226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,4,128,1,float16,fp8,0,0.057633602619171144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,4,128,1,fp8,fp8,0,0.05763520002365112
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,8,128,1,float16,float16,0,0.05773599743843079
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,8,128,1,float16,fp8,0,0.059201598167419434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,1,128,1,float16,fp8,0,0.19455039501190186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,64,128,1,float16,float16,0,0.06671839952468872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,64,128,1,float16,fp8,0,0.05987840294837952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,1,128,1,fp8,fp8,0,0.19111679792404174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,1,128,1,float16,fp8,0,0.029315200448036195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,1,128,1,fp8,fp8,0,0.02935200035572052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,2,128,1,float16,float16,0,0.02741599977016449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,2,128,1,float16,fp8,0,0.0295199990272522
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,2,128,1,fp8,fp8,0,0.029572799801826477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,4,128,1,float16,float16,0,0.027764800190925597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,64,2,128,1,float16,fp8,0,0.10401279926300049
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,4,128,1,float16,fp8,0,0.02961759865283966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,4,128,1,fp8,fp8,0,0.029556798934936523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,8,128,1,float16,float16,0,0.02945759892463684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,8,128,1,float16,fp8,0,0.0296207994222641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,8,128,1,fp8,fp8,0,0.02956799864768982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,64,128,1,fp8,fp8,0,0.11352800130844116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,64,128,1,float16,float16,0,0.03677760064601898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,64,128,1,float16,fp8,0,0.030131199955940248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,64,128,1,fp8,fp8,0,0.030276799201965333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,1,128,1,float16,float16,0,0.017582400143146514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,1,128,1,float16,fp8,0,0.01892800033092499
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,1,128,1,fp8,fp8,0,0.018828800320625304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,2,128,1,float16,float16,0,0.017608000338077544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,2,128,1,float16,fp8,0,0.01890240013599396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,2,128,1,fp8,fp8,0,0.018880000710487364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,4,128,1,float16,float16,0,0.017918400466442108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,4,128,1,float16,fp8,0,0.018960000574588777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,4,128,1,fp8,fp8,0,0.019020800292491914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,8,128,1,fp8,fp8,0,0.05925120115280151
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,8,128,1,float16,float16,0,0.018182399868965148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,8,128,1,float16,fp8,0,0.01913280040025711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,64,8,128,1,fp8,fp8,0,0.01897599995136261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,64,128,1,float16,float16,0,0.018756799399852753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,64,128,1,fp8,fp8,0,0.018801599740982056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,1,128,1,float16,float16,0,0.01281760036945343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,1,128,1,float16,fp8,0,0.013449600338935852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,1,128,1,fp8,fp8,0,0.013475200533866883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,2,128,1,float16,float16,0,0.012678399682044983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,2,128,1,float16,fp8,0,0.013512000441551208
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,2,128,1,fp8,fp8,0,0.01340319961309433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,4,128,1,float16,float16,0,0.013022400438785553
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,4,128,1,float16,fp8,0,0.01342719942331314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,4,128,1,fp8,fp8,0,0.013638399541378021
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,8,128,1,float16,float16,0,0.013064000010490417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,8,128,1,float16,fp8,0,0.013836799561977387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,64,128,1,float16,float16,0,0.014091199636459351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,64,128,1,float16,fp8,0,0.013942399621009826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,64,128,1,fp8,fp8,0,0.013899199664592743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,1,128,1,float16,float16,0,0.012388800084590913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,1,128,1,float16,fp8,0,0.013174399733543396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,1,128,1,fp8,fp8,0,0.013176000118255616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,2,128,1,float16,float16,0,0.012518399953842163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,2,128,1,float16,fp8,0,0.013476799428462981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,2,128,1,fp8,fp8,0,0.013364799320697784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,4,128,1,float16,float16,0,0.01265760064125061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,4,128,1,float16,fp8,0,0.01321599930524826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,4,128,1,fp8,fp8,0,0.0131632000207901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,8,128,1,float16,float16,0,0.012545600533485413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,64,2,128,1,float16,float16,0,0.05440160036087036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,8,128,1,float16,fp8,0,0.013076800107955932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,64,8,128,1,fp8,fp8,0,0.01348160058259964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,64,128,1,float16,float16,0,0.013748799264431
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,64,128,1,float16,fp8,0,0.013633599877357483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,64,128,1,fp8,fp8,0,0.013732799887657165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,1,128,1,float16,float16,0,0.012479999661445617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,1,128,1,float16,fp8,0,0.013079999387264252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,1,128,1,fp8,fp8,0,0.013092799484729767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,2,128,1,float16,float16,0,0.012508800625801087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,2,128,1,float16,fp8,0,0.013081599771976472
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,64,1,128,1,float16,float16,0,0.17998080253601073
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,4,128,1,float16,float16,0,0.012222400307655335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,4,128,1,float16,fp8,0,0.013073599338531494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,4,128,1,fp8,fp8,0,0.012883199751377106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,64,128,1,float16,fp8,0,0.018939200043678283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,8,128,1,float16,fp8,0,0.012887999415397644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,8,128,1,fp8,fp8,0,0.013014400005340576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,64,1,128,1,float16,float16,0,0.17929600477218627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,64,1,128,1,float16,fp8,0,0.19203519821166992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,64,1,128,1,fp8,fp8,0,0.19383200407028198
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,64,2,128,1,float16,float16,0,0.18156960010528564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,64,8,128,1,fp8,fp8,0,0.01374559998512268
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,64,2,128,1,float16,fp8,0,0.19318560361862183
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,64,2,128,1,fp8,fp8,0,0.19398080110549926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,64,4,128,1,float16,float16,0,0.18622879981994628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,64,4,128,1,float16,fp8,0,0.19484959840774535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,64,4,128,1,fp8,fp8,0,0.19608160257339477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,64,8,128,1,float16,float16,0,0.19296319484710694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,64,8,128,1,float16,fp8,0,0.20307040214538574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,64,8,128,1,fp8,fp8,0,0.2033344030380249
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,64,128,1,float16,float16,0,0.3429199934005737
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,64,128,1,float16,fp8,0,0.3549855947494507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,1,128,1,float16,float16,0,0.09548640251159668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,64,128,1,fp8,fp8,0,0.3575263977050781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,1,128,1,float16,fp8,0,0.10363999605178834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,1,128,1,fp8,fp8,0,0.1042464017868042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,2,128,1,float16,float16,0,0.09688959717750549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,2,128,1,float16,fp8,0,0.10585440397262573
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,2,128,1,fp8,fp8,0,0.10403200387954711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,4,128,1,float16,float16,0,0.09849439859390259
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,4,128,1,fp8,fp8,0,0.10613119602203369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,8,128,1,float16,float16,0,0.10106559991836547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,8,128,1,float16,fp8,0,0.10797920227050781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,8,128,1,fp8,fp8,0,0.10908160209655762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,64,128,1,float16,float16,0,0.1777680039405823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,64,128,1,float16,fp8,0,0.18574399948120118
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,64,128,1,fp8,fp8,0,0.1871135950088501
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,2,128,1,fp8,fp8,0,0.013206399977207184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,1,128,1,float16,fp8,0,0.05693920254707337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,1,128,1,fp8,fp8,0,0.05692800283432007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,2,128,1,float16,float16,0,0.05591999888420105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,2,128,1,float16,fp8,0,0.05748639702796936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,64,128,1,fp8,fp8,0,0.059988802671432494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,2,128,1,fp8,fp8,0,0.05753440260887146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,4,128,1,float16,float16,0,0.056523197889328004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,4,128,1,float16,fp8,0,0.058329600095748904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,4,128,1,fp8,fp8,0,0.05792800188064575
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,8,128,1,float16,float16,0,0.058513599634170535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,8,128,1,float16,fp8,0,0.0592415988445282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,8,128,1,fp8,fp8,0,0.059935998916625974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,64,128,1,float16,float16,0,0.09589120149612426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,64,128,1,float16,fp8,0,0.09792159795761109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,1,128,1,float16,float16,0,0.027534401416778563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,1,128,1,float16,fp8,0,0.02956480085849762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,1,128,1,fp8,fp8,0,0.02933279871940613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,2,128,1,float16,float16,0,0.027798399329185486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,2,128,1,float16,fp8,0,0.02935360074043274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,2,128,1,fp8,fp8,0,0.029758399724960326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,4,128,1,float16,float16,0,0.028169599175453187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,4,128,1,float16,fp8,0,0.02962239980697632
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,4,128,1,fp8,fp8,0,0.029718399047851562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,8,128,1,float16,float16,0,0.03006879985332489
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,8,128,1,float16,fp8,0,0.029963201284408568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,8,128,1,fp8,fp8,0,0.02991360127925873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,64,128,1,float16,float16,0,0.05189120173454285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,64,128,1,float16,fp8,0,0.04809600114822388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,64,128,1,fp8,fp8,0,0.04832639992237091
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,1,128,1,float16,float16,0,0.017500799894332886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,1,128,1,float16,fp8,0,0.018849599361419677
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,1,128,1,fp8,fp8,0,0.018880000710487364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,2,128,1,float16,float16,0,0.018078400194644927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,2,128,1,float16,fp8,0,0.018828800320625304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,2,128,1,fp8,fp8,0,0.019100800156593323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,4,128,1,float16,float16,0,0.017815999686717987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,4,128,1,float16,fp8,0,0.019017599523067474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,4,128,1,fp8,fp8,0,0.018931199610233308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,8,128,1,float16,float16,0,0.01849119961261749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,8,128,1,float16,fp8,0,0.01896799951791763
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,64,8,128,1,fp8,fp8,0,0.019142399728298187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,64,128,1,float16,float16,0,0.02659200131893158
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,64,128,1,float16,fp8,0,0.028003200888633728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,64,128,1,fp8,fp8,0,0.028121599555015565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,1,128,1,float16,float16,0,0.012779200077056884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,1,128,1,float16,fp8,0,0.013711999356746673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,1,128,1,fp8,fp8,0,0.013532799482345582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,2,128,1,float16,float16,0,0.012969599664211273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,2,128,1,float16,fp8,0,0.013495999574661254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,2,128,1,fp8,fp8,0,0.01369439959526062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,64,1,128,1,float16,float16,0,0.054681599140167236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,4,128,1,fp8,fp8,0,0.013812799751758576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,8,128,1,float16,float16,0,0.013369600474834441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,64,1,128,1,float16,float16,0,0.02725279927253723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,8,128,1,float16,fp8,0,0.013857600092887879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,8,128,1,fp8,fp8,0,0.013644799590110779
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,64,128,1,float16,fp8,0,0.018228800594806673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,64,128,1,fp8,fp8,0,0.018214400112628936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,1,128,1,float16,float16,0,0.012513600289821625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,64,64,128,1,fp8,fp8,0,0.09796479940414429
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,1,128,1,fp8,fp8,0,0.013220800459384919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,2,128,1,float16,float16,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,2,128,1,float16,fp8,0,0.013286399841308593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,2,128,1,fp8,fp8,0,0.013185599446296692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,4,128,1,float16,float16,0,0.012691199779510498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,4,128,1,float16,fp8,0,0.013172799348831176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,4,128,1,fp8,fp8,0,0.01311040073633194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,8,128,1,float16,float16,0,0.012455999851226807
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,8,128,1,float16,fp8,0,0.013126400113105775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,8,128,1,fp8,fp8,0,0.013076800107955932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,64,128,1,float16,float16,0,0.013704000413417817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,64,128,1,float16,fp8,0,0.013766400516033173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,64,128,1,fp8,fp8,0,0.013872000575065612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,64,4,128,1,float16,fp8,0,0.10633599758148193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,1,128,1,float16,fp8,0,0.013257600367069244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,2,128,1,float16,float16,0,0.012433599680662155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,2,128,1,float16,fp8,0,0.013120000064373017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,2,128,1,fp8,fp8,0,0.013107199966907502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,4,128,1,float16,float16,0,0.012451200187206269
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,4,128,1,float16,float16,0,0.012878400087356568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,4,128,1,float16,fp8,0,0.013070400059223174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,4,128,1,fp8,fp8,0,0.013091200590133667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,8,128,1,float16,float16,0,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,8,128,1,float16,fp8,0,0.013120000064373017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,8,128,1,fp8,fp8,0,0.013027200102806091
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,64,128,1,float16,float16,0,0.013595199584960938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,64,128,1,float16,float16,0,0.01764799952507019
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,64,128,1,fp8,fp8,0,0.013532799482345582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,1,128,1,float16,float16,0,0.012145599722862244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,64,1,128,1,float16,fp8,0,0.013550400733947754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,1,128,1,fp8,fp8,0,0.012838399410247803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,2,128,1,float16,float16,0,0.012148799747228623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,2,128,1,float16,fp8,0,0.012894399464130402
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,2,128,1,fp8,fp8,0,0.012859199941158295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,4,128,1,float16,float16,0,0.012225600332021714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,4,128,1,float16,fp8,0,0.013094399869441987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,4,128,1,fp8,fp8,0,0.012908799946308136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,8,128,1,float16,float16,0,0.012161599844694138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,8,128,1,float16,fp8,0,0.012830400466918945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,8,128,1,fp8,fp8,0,0.012729600071907043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,64,1,128,1,float16,float16,0,0.09505919814109802
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,64,1,128,1,float16,fp8,0,0.10616639852523804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,64,1,128,1,fp8,fp8,0,0.10460000038146973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,64,2,128,1,float16,float16,0,0.0961184024810791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,64,2,128,1,float16,fp8,0,0.10532159805297851
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,64,2,128,1,fp8,fp8,0,0.10537120103836059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,1,128,1,float16,float16,0,0.0125231996178627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,64,1,128,1,fp8,fp8,0,0.013102400302886962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,64,4,128,1,float16,fp8,0,0.1064303994178772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,64,4,128,1,fp8,fp8,0,0.1068992018699646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,64,8,128,1,float16,float16,0,0.10310879945755005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,64,4,128,1,float16,fp8,0,0.013894400000572205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,64,8,128,1,float16,fp8,0,0.109278404712677
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,64,8,128,1,float16,float16,0,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,64,128,1,float16,fp8,0,0.013600000739097595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,64,128,1,float16,float16,0,0.300817608833313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,64,128,1,float16,fp8,0,0.33489279747009276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,1,128,1,float16,fp8,0,0.05834239721298218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,64,128,1,fp8,fp8,0,0.3347840070724487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,2,128,1,float16,float16,0,0.05711680054664612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,2,128,1,float16,fp8,0,0.05846400260925293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,2,128,1,fp8,fp8,0,0.05849760174751282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,4,128,1,float16,float16,0,0.058303999900817874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,4,128,1,float16,fp8,0,0.058827197551727294
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,4,128,1,fp8,fp8,0,0.05905119776725769
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,8,128,1,float16,float16,0,0.0599839985370636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,8,128,1,float16,fp8,0,0.060545599460601805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,8,128,1,fp8,fp8,0,0.06051040291786194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,64,128,1,float16,float16,0,0.15678080320358276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,64,128,1,float16,fp8,0,0.17326560020446777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,1,128,1,float16,float16,0,0.028040000796318056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,64,128,1,fp8,fp8,0,0.17437920570373536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,1,128,1,float16,fp8,0,0.02990719974040985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,1,128,1,fp8,fp8,0,0.0298335999250412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,2,128,1,float16,float16,0,0.02812800109386444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,2,128,1,float16,fp8,0,0.029820799827575684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,2,128,1,fp8,fp8,0,0.029916799068450926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,4,128,1,float16,float16,0,0.028204798698425293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,4,128,1,float16,fp8,0,0.03001439869403839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,4,128,1,fp8,fp8,0,0.029791998863220214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,8,128,1,float16,float16,0,0.02996160089969635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,8,128,1,float16,fp8,0,0.030059200525283814
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,64,8,128,1,fp8,fp8,0,0.03019680082798004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,64,4,128,1,float16,float16,0,0.09873600006103515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,64,128,1,float16,fp8,0,0.0853712022304535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,64,128,1,fp8,fp8,0,0.08554080128669739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,1,128,1,float16,float16,0,0.01807519942522049
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,1,128,1,float16,fp8,0,0.01929280012845993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,1,128,1,fp8,fp8,0,0.01932000070810318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,2,128,1,float16,float16,0,0.017990399897098542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,2,128,1,float16,fp8,0,0.0192208006978035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,2,128,1,fp8,fp8,0,0.01929280012845993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,4,128,1,float16,float16,0,0.018086400628089905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,4,128,1,float16,fp8,0,0.019303999841213226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,64,8,128,1,fp8,fp8,0,0.10861120223999024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,4,128,1,fp8,fp8,0,0.01926880031824112
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,8,128,1,float16,float16,0,0.018249599635601042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,8,128,1,fp8,fp8,0,0.019227199256420135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,8,128,1,float16,fp8,0,0.01929759979248047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,64,128,1,float16,fp8,0,0.04640159904956818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,64,128,1,fp8,fp8,0,0.04642240107059479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,1,128,1,float16,float16,0,0.05594879984855652
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,1,128,1,float16,fp8,0,0.013795199990272521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,1,128,1,fp8,fp8,0,0.013846400380134582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,2,128,1,float16,float16,0,0.013305599987506866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,2,128,1,float16,fp8,0,0.013577599823474885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,2,128,1,fp8,fp8,0,0.013948799669742584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,4,128,1,float16,float16,0,0.012985600531101227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,4,128,1,float16,fp8,0,0.01385280042886734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,4,128,1,fp8,fp8,0,0.01361120045185089
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,8,128,1,float16,float16,0,0.013387200236320496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,8,128,1,float16,fp8,0,0.013758400082588195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,64,128,1,float16,float16,0,0.02543199956417084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,64,128,1,float16,fp8,0,0.027456000447273254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,64,128,1,fp8,fp8,0,0.027401599287986755
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,1,128,1,float16,float16,0,0.012489599734544754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,1,128,1,float16,fp8,0,0.013449600338935852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,1,128,1,fp8,fp8,0,0.013575999438762665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,2,128,1,float16,float16,0,0.012600000202655792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,2,128,1,float16,fp8,0,0.013483199477195739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,2,128,1,fp8,fp8,0,0.013383999466896057
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,4,128,1,float16,float16,0,0.012969599664211273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,4,128,1,float16,fp8,0,0.013331200182437896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,4,128,1,fp8,fp8,0,0.013409599661827087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,8,128,1,float16,float16,0,0.012880000472068786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,8,128,1,float16,fp8,0,0.013372799754142762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,64,8,128,1,fp8,fp8,0,0.013436800241470337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,64,128,1,float16,float16,0,0.017324799299240114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,64,128,1,float16,fp8,0,0.018177600204944612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,64,128,1,fp8,fp8,0,0.018089599907398224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,1,128,1,float16,fp8,0,0.01305440068244934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,1,128,1,fp8,fp8,0,0.012972800433635712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,2,128,1,float16,float16,0,0.01250240057706833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,2,128,1,float16,fp8,0,0.013116799294948578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,64,64,128,1,float16,float16,0,0.08439520001411438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,2,128,1,fp8,fp8,0,0.013011200726032257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,4,128,1,float16,float16,0,0.012392000108957291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,4,128,1,float16,fp8,0,0.012868799269199371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,4,128,1,fp8,fp8,0,0.012999999523162841
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,8,128,1,float16,fp8,0,0.013121600449085235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,8,128,1,fp8,fp8,0,0.013638399541378021
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,64,128,1,float16,float16,0,0.013689599931240082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,64,128,1,float16,float16,0,0.04140639901161194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,64,128,1,float16,fp8,0,0.013510400056838989
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,1,128,1,float16,float16,0,0.013169600069522858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,64,128,1,fp8,fp8,0,0.013596799969673157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,1,128,1,float16,float16,0,0.012228800356388092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,1,128,1,float16,fp8,0,0.012827199697494508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,1,128,1,fp8,fp8,0,0.012859199941158295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,2,128,1,float16,fp8,0,0.013006399571895599
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,2,128,1,fp8,fp8,0,0.012988799810409546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,4,128,1,float16,float16,0,0.012342400103807449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,64,8,128,1,fp8,fp8,0,0.013660800457000733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,4,128,1,float16,fp8,0,0.013022400438785553
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,4,128,1,fp8,fp8,0,0.012715199589729309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,8,128,1,float16,fp8,0,0.01279039978981018
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,8,128,1,fp8,fp8,0,0.012929600477218629
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,64,128,1,float16,float16,0,0.013544000685214996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,64,128,1,float16,fp8,0,0.013491199910640716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,64,128,1,fp8,fp8,0,0.013523200154304504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,1,128,1,float16,float16,0,0.012012799829244613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,1,128,1,float16,fp8,0,0.012638400495052337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,1,128,1,fp8,fp8,0,0.012643200159072877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,2,128,1,float16,float16,0,0.012035199999809265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,2,128,1,float16,fp8,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,2,128,1,fp8,fp8,0,0.0126351997256279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,4,128,1,float16,float16,0,0.012150400131940842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,4,128,1,float16,fp8,0,0.012691199779510498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,4,128,1,fp8,fp8,0,0.012683199346065521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,8,128,1,float16,float16,0,0.012039999663829803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,8,128,1,float16,float16,0,0.01231520026922226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,8,128,1,fp8,fp8,0,0.01255200058221817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,64,1,128,1,float16,float16,0,0.020848000049591066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,64,1,128,1,float16,fp8,0,0.022735999524593355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,64,1,128,1,fp8,fp8,0,0.02277279943227768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,64,2,128,1,float16,float16,0,0.02831200063228607
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,64,2,128,1,float16,fp8,0,0.031455999612808226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,64,2,128,1,fp8,fp8,0,0.03145439922809601
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,64,4,128,1,float16,float16,0,0.04315040111541748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,64,1,128,1,float16,fp8,0,0.012878400087356568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,64,4,128,1,fp8,fp8,0,0.04913919866085052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,64,8,128,1,float16,float16,0,0.07079200148582458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,64,8,128,1,float16,fp8,0,0.08419200181961059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,64,8,128,1,fp8,fp8,0,0.08466719985008239
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,64,128,1,float16,float16,0,0.2382672071456909
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,64,1,128,1,float16,float16,0,0.012280000001192093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,64,128,1,float16,fp8,0,0.29687840938568116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,1,128,1,float16,float16,0,0.01515199989080429
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,64,128,1,fp8,fp8,0,0.2976608037948608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,1,128,1,fp8,fp8,0,0.015860800445079804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,2,128,1,float16,float16,0,0.018318399786949158
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,2,128,1,float16,fp8,0,0.02014400064945221
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,2,128,1,fp8,fp8,0,0.01990559995174408
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,64,8,128,1,float16,fp8,0,0.01268800050020218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,4,128,1,float16,fp8,0,0.028996801376342772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,4,128,1,fp8,fp8,0,0.029356798529624938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,8,128,1,float16,float16,0,0.04053440093994141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,8,128,1,float16,fp8,0,0.04690400063991547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,8,128,1,fp8,fp8,0,0.04685280025005341
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,64,128,1,float16,float16,0,0.12720799446105957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,64,128,1,float16,fp8,0,0.15538239479064941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,1,128,1,float16,float16,0,0.013310399651527405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,64,128,1,fp8,fp8,0,0.15535680055618287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,1,128,1,float16,fp8,0,0.014390400052070618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,1,128,1,fp8,fp8,0,0.014139199256896972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,2,128,1,float16,float16,0,0.013492800295352936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,64,1,128,1,fp8,fp8,0,0.05793600082397461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,2,128,1,fp8,fp8,0,0.014582400023937226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,2,128,1,float16,float16,0,0.012150400131940842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,4,128,1,float16,float16,0,0.01717440038919449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,64,4,128,1,float16,fp8,0,0.0489760011434555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,8,128,1,float16,float16,0,0.024417600035667418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,8,128,1,float16,fp8,0,0.02736319899559021
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,64,128,1,float16,float16,0,0.07017599940299987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,64,8,128,1,float16,float16,0,0.012323199957609176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,64,128,1,float16,fp8,0,0.08222399950027466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,1,128,1,float16,float16,0,0.012356799840927125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,1,128,1,float16,fp8,0,0.013305599987506866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,1,128,1,fp8,fp8,0,0.013407999277114868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,2,128,1,float16,float16,0,0.012638400495052337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,2,128,1,float16,fp8,0,0.013582399487495423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,2,128,1,fp8,fp8,0,0.013489599525928497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,4,128,1,float16,float16,0,0.012646399438381195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,4,128,1,float16,fp8,0,0.01366720050573349
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,4,128,1,fp8,fp8,0,0.013337600231170654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,8,128,1,float16,float16,0,0.01635040044784546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,8,128,1,float16,fp8,0,0.017759999632835387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,1,128,1,float16,fp8,0,0.016044799983501435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,8,128,1,fp8,fp8,0,0.01773280054330826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,64,128,1,float16,float16,0,0.04036000072956085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,64,128,1,float16,fp8,0,0.04561600089073181
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,64,4,128,1,float16,float16,0,0.026051199436187743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,1,128,1,float16,float16,0,0.012088000029325485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,1,128,1,float16,fp8,0,0.01273919939994812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,1,128,1,fp8,fp8,0,0.01286720037460327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,2,128,1,float16,float16,0,0.012240000069141388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,2,128,1,float16,fp8,0,0.012992000579833985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,2,128,1,fp8,fp8,0,0.01313599944114685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,4,128,1,float16,float16,0,0.012240000069141388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,4,128,1,float16,fp8,0,0.013014400005340576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,4,128,1,fp8,fp8,0,0.01287200003862381
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,8,128,1,float16,float16,0,0.012355200201272964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,8,128,1,float16,fp8,0,0.013011200726032257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,8,128,1,fp8,fp8,0,0.012982399761676788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,64,128,1,float16,float16,0,0.025196799635887147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,64,128,1,float16,fp8,0,0.027267199754714967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,64,128,1,fp8,fp8,0,0.027164798974990845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,4,128,1,float16,fp8,0,0.019049599766731262
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,1,128,1,float16,float16,0,0.011713600158691407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,4,128,1,fp8,fp8,0,0.01871519982814789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,1,128,1,float16,fp8,0,0.012603199481964112
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,2,128,1,float16,float16,0,0.011593600362539291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,2,128,1,float16,fp8,0,0.012689599394798278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,2,128,1,fp8,fp8,0,0.01236959993839264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,4,128,1,float16,float16,0,0.011955200135707856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,4,128,1,float16,fp8,0,0.01247360035777092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,4,128,1,fp8,fp8,0,0.012600000202655792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,8,128,1,float16,float16,0,0.011843200027942657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,8,128,1,float16,fp8,0,0.012614400684833526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,64,64,128,1,fp8,fp8,0,0.08218560218811036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,64,128,1,float16,float16,0,0.017092800140380858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,64,128,1,float16,fp8,0,0.017683200538158417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,64,128,1,fp8,fp8,0,0.017696000635623932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,1,128,1,float16,float16,0,0.011521600186824799
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,1,128,1,float16,fp8,0,0.012247999757528305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,1,128,1,fp8,fp8,0,0.012361600250005721
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,2,128,1,float16,float16,0,0.011611200124025344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,2,128,1,float16,fp8,0,0.01228479966521263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,2,128,1,fp8,fp8,0,0.012457600235939026
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,4,128,1,float16,float16,0,0.011641599982976914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,4,128,1,float16,fp8,0,0.012638400495052337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,4,128,1,fp8,fp8,0,0.012409599870443344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,8,128,1,float16,float16,0,0.011947199702262878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,8,128,1,float16,fp8,0,0.012736000120639801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,64,8,128,1,fp8,fp8,0,0.012718400359153748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,64,128,1,float16,float16,0,0.013497599959373474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,64,128,1,float16,fp8,0,0.013545599579811097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,64,128,1,fp8,fp8,0,0.013635200262069703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,1,128,1,float16,float16,0,0.011401599645614624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,1,128,1,float16,fp8,0,0.012308800220489502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,1,128,1,fp8,fp8,0,0.01215839982032776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,2,128,1,float16,float16,0,0.011473599821329117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,2,128,1,float16,fp8,0,0.012267199903726577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,2,128,1,fp8,fp8,0,0.012204799801111221
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,4,128,1,float16,float16,0,0.011665599793195725
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,4,128,1,float16,fp8,0,0.012120000272989272
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,8,128,1,float16,float16,0,0.01162080019712448
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,64,64,128,1,fp8,fp8,0,0.04549280107021332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,8,128,1,fp8,fp8,0,0.01236959993839264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,64,128,1,float16,float16,0,0.01311040073633194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,64,128,1,float16,fp8,0,0.013380800187587739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,64,128,1,fp8,fp8,0,0.013380800187587739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,1,128,1,float16,float16,0,0.011529599875211715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,1,128,1,float16,fp8,0,0.012161599844694138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,1,128,1,fp8,fp8,0,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,2,128,1,float16,fp8,0,0.01422560065984726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,2,128,1,float16,fp8,0,0.01225920021533966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,2,128,1,fp8,fp8,0,0.01223199963569641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,1,128,1,fp8,fp8,0,0.012411200255155564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,64,8,128,1,fp8,fp8,0,0.027371200919151305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,4,128,1,fp8,fp8,0,0.012289600074291229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,8,128,1,float16,float16,0,0.011633600294589996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,64,8,128,1,fp8,fp8,0,0.012881599366664886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,8,128,1,fp8,fp8,0,0.01228799968957901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,4,128,1,fp8,fp8,0,0.012385600060224534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,64,8,128,1,float16,fp8,0,0.012374400347471236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,2,128,1,float16,float16,0,0.011360000073909759
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,4,128,1,float16,float16,0,0.011419200152158738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,4,128,1,float16,fp8,0,0.012088000029325485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,64,8,128,1,float16,fp8,0,0.012412799894809723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,48,1,128,1,float16,fp8,0,39.08258972167969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,48,1,128,1,fp8,fp8,0,39.638815307617186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,48,2,128,1,float16,fp8,0,39.59253845214844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,48,4,128,1,float16,fp8,0,39.21055603027344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,48,2,128,1,fp8,fp8,0,39.29718933105469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,48,1,128,1,float16,float16,0,47.357241821289065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,48,2,128,1,float16,float16,0,46.402651977539065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,48,4,128,1,float16,float16,0,46.81326599121094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,48,128,1,float16,float16,0,23.862094116210937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,1,128,1,float16,float16,0,22.89574432373047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,48,4,128,1,fp8,fp8,0,39.256222534179685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,48,8,128,1,fp8,fp8,0,39.3524658203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,48,8,128,1,float16,fp8,0,39.94301147460938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,48,8,128,1,float16,float16,0,47.92995910644531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,1,128,1,float16,fp8,0,19.936688232421876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,1,128,1,fp8,fp8,0,19.34931640625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,2,128,1,float16,fp8,0,19.399037170410157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,2,128,1,fp8,fp8,0,19.698393249511717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,2,128,1,float16,float16,0,22.982054138183592
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,4,128,1,float16,fp8,0,19.534034729003906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,4,128,1,fp8,fp8,0,19.711912536621092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,4,128,1,float16,float16,0,23.783103942871094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,8,128,1,float16,fp8,0,20.247357177734376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,48,128,1,float16,float16,0,12.839703369140626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,8,128,1,fp8,fp8,0,19.762936401367188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,8,128,1,float16,float16,0,23.944747924804688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,1,128,1,fp8,fp8,0,9.611737823486328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,1,128,1,float16,fp8,0,9.586270141601563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,1,128,1,float16,float16,0,11.920331573486328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,2,128,1,float16,float16,0,11.491120147705079
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,2,128,1,fp8,fp8,0,9.660829162597656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,2,128,1,float16,fp8,0,9.830124664306641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,4,128,1,float16,float16,0,12.007364654541016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,4,128,1,float16,fp8,0,9.639518737792969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,4,128,1,fp8,fp8,0,9.79249267578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,8,128,1,float16,fp8,0,9.876847839355468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,8,128,1,float16,float16,0,11.89685287475586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,48,128,1,float16,float16,0,6.534278106689453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,8,128,1,fp8,fp8,0,9.644659423828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,1,128,1,float16,float16,0,5.94085922241211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,1,128,1,float16,fp8,0,4.752795028686523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,1,128,1,fp8,fp8,0,4.860649490356446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,2,128,1,float16,float16,0,5.686827087402344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,2,128,1,float16,fp8,0,4.817323303222656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,2,128,1,fp8,fp8,0,4.894903945922851
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,4,128,1,float16,float16,0,5.706342315673828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,4,128,1,float16,fp8,0,4.807416152954102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,4,128,1,fp8,fp8,0,4.971688079833984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,8,128,1,float16,float16,0,6.347649765014649
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,8,128,1,float16,fp8,0,4.847116851806641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,8,128,1,fp8,fp8,0,5.019728088378907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,48,1,128,1,float16,fp8,0,21.931932067871095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,48,1,128,1,float16,float16,0,26.823876953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,48,128,1,float16,fp8,0,4.8388416290283205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,48,48,128,1,fp8,fp8,0,4.801067352294922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,48,128,1,float16,fp8,0,9.831626892089844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,48,48,128,1,fp8,fp8,0,10.008470153808593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,48,1,128,1,fp8,fp8,0,22.67421112060547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,48,128,1,float16,fp8,0,19.955810546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,48,48,128,1,fp8,fp8,0,19.753744506835936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,48,2,128,1,float16,float16,0,26.849514770507813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,48,2,128,1,float16,fp8,0,21.920709228515626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,48,2,128,1,fp8,fp8,0,22.089939880371094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,48,4,128,1,float16,float16,0,26.956265258789063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,48,4,128,1,float16,fp8,0,22.13945770263672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,48,128,1,float16,fp8,0,12.323700714111329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,48,4,128,1,fp8,fp8,0,21.976158142089844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,48,128,1,float16,float16,0,15.008145141601563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,48,8,128,1,float16,fp8,0,22.437261962890624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,48,8,128,1,fp8,fp8,0,22.186846923828124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,48,8,128,1,float16,float16,0,28.10433349609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,48,128,1,fp8,fp8,0,11.373729705810547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,1,128,1,float16,float16,0,13.84488525390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,1,128,1,float16,fp8,0,11.061637115478515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,1,128,1,fp8,fp8,0,11.001156616210938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,2,128,1,float16,float16,0,13.010633850097657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,2,128,1,float16,fp8,0,10.96702880859375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,2,128,1,fp8,fp8,0,11.391582489013672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,4,128,1,float16,fp8,0,10.967865753173829
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,4,128,1,float16,float16,0,13.260942077636718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,48,128,1,float16,fp8,0,5.951723098754883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,48,128,1,float16,float16,0,7.039630126953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,4,128,1,fp8,fp8,0,11.08475341796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,8,128,1,float16,fp8,0,11.41397476196289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,8,128,1,float16,float16,0,13.511224365234375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,48,128,1,fp8,fp8,0,5.648260879516601
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,48,8,128,1,fp8,fp8,0,11.006827545166015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,1,128,1,float16,float16,0,6.5490974426269535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,1,128,1,float16,fp8,0,5.502441787719727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,1,128,1,fp8,fp8,0,5.492588806152344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,2,128,1,float16,float16,0,6.463078308105469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,2,128,1,fp8,fp8,0,5.6088417053222654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,4,128,1,float16,fp8,0,5.563000106811524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,4,128,1,float16,float16,0,6.937318420410156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,4,128,1,fp8,fp8,0,5.712115097045898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,8,128,1,float16,fp8,0,5.478231811523438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,8,128,1,float16,float16,0,6.7558753967285154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,48,128,1,float16,float16,0,3.674697494506836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,8,128,1,fp8,fp8,0,5.66033935546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,48,128,1,fp8,fp8,0,3.2221439361572264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,1,128,1,float16,float16,0,3.364744186401367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,48,2,128,1,float16,fp8,0,5.477825546264649
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,1,128,1,fp8,fp8,0,2.873262405395508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,2,128,1,float16,fp8,0,2.7704751968383787
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,2,128,1,float16,float16,0,3.4465808868408203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,2,128,1,fp8,fp8,0,2.7553728103637694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,4,128,1,float16,fp8,0,2.687017631530762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,4,128,1,float16,float16,0,3.3538223266601563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,1,128,1,float16,fp8,0,2.706564712524414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,8,128,1,float16,float16,0,3.4698177337646485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,8,128,1,float16,fp8,0,2.9277599334716795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,8,128,1,fp8,fp8,0,2.7202480316162108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,48,128,1,float16,fp8,0,2.906358337402344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,48,4,128,1,fp8,fp8,0,2.749959945678711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,48,1,128,1,float16,fp8,0,15.504141235351563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,48,1,128,1,fp8,fp8,0,15.145179748535156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,48,2,128,1,fp8,fp8,0,15.430819702148437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,48,1,128,1,float16,float16,0,18.314414978027344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,48,4,128,1,float16,fp8,0,15.512799072265626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,48,4,128,1,float16,float16,0,18.263973999023438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,48,2,128,1,float16,float16,0,18.995626831054686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,48,2,128,1,float16,fp8,0,15.635289001464844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,48,128,1,float16,fp8,0,8.080684661865234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,48,128,1,fp8,fp8,0,8.100415802001953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,48,128,1,float16,float16,0,9.753385925292969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,1,128,1,float16,float16,0,9.17194061279297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,48,4,128,1,fp8,fp8,0,15.78675537109375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,48,8,128,1,float16,fp8,0,15.885054016113282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,48,8,128,1,fp8,fp8,0,16.02802429199219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,48,8,128,1,float16,float16,0,18.66853485107422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,1,128,1,float16,fp8,0,7.809645080566407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,1,128,1,fp8,fp8,0,7.774681854248047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,2,128,1,float16,fp8,0,7.840169525146484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,2,128,1,float16,float16,0,9.736860656738282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,2,128,1,fp8,fp8,0,8.169544219970703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,4,128,1,float16,fp8,0,7.675313568115234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,4,128,1,float16,float16,0,9.46578369140625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,4,128,1,fp8,fp8,0,7.777315521240235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,48,128,1,float16,float16,0,5.070108795166016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,8,128,1,float16,fp8,0,7.852603149414063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,8,128,1,float16,float16,0,9.531505584716797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,48,128,1,fp8,fp8,0,4.107440185546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,1,128,1,float16,float16,0,4.480558395385742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,48,8,128,1,fp8,fp8,0,7.836032104492188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,1,128,1,float16,fp8,0,3.821521759033203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,1,128,1,fp8,fp8,0,4.012516784667969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,48,128,1,float16,fp8,0,3.981342315673828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,2,128,1,float16,fp8,0,3.938731384277344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,2,128,1,float16,float16,0,4.730238342285157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,4,128,1,float16,fp8,0,3.8623294830322266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,4,128,1,float16,float16,0,4.5119281768798825
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,4,128,1,fp8,fp8,0,3.8576896667480467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,48,128,1,float16,float16,0,2.544987106323242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,8,128,1,float16,float16,0,4.546868896484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,48,128,1,float16,fp8,0,2.021830368041992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,8,128,1,float16,fp8,0,3.8188751220703123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,48,128,1,fp8,fp8,0,2.147862434387207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,8,128,1,fp8,fp8,0,3.831012725830078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,48,2,128,1,fp8,fp8,0,3.831195068359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,1,128,1,float16,float16,0,2.0550527572631836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,1,128,1,fp8,fp8,0,2.072921562194824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,2,128,1,float16,float16,0,2.0528160095214845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,1,128,1,float16,fp8,0,2.347761535644531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,2,128,1,fp8,fp8,0,1.9270063400268556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,4,128,1,float16,fp8,0,1.888582420349121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,4,128,1,float16,float16,0,2.201479911804199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,4,128,1,fp8,fp8,0,2.028278350830078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,8,128,1,float16,fp8,0,2.0922975540161133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,8,128,1,fp8,fp8,0,1.9965839385986328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,2,128,1,float16,fp8,0,1.8942703247070312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,48,8,128,1,float16,float16,0,2.180780792236328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,48,1,128,1,float16,fp8,0,19.955465698242186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,48,1,128,1,fp8,fp8,0,20.084474182128908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,48,2,128,1,float16,fp8,0,20.790306091308594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,48,2,128,1,fp8,fp8,0,20.777862548828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,48,1,128,1,float16,float16,0,24.008114624023438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,48,2,128,1,float16,float16,0,25.04183349609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,48,4,128,1,float16,fp8,0,20.290487670898436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,48,4,128,1,float16,float16,0,23.87122039794922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,48,128,1,float16,fp8,0,10.607065582275391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,48,128,1,float16,float16,0,12.978582763671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,48,128,1,fp8,fp8,0,10.599630737304688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,1,128,1,float16,float16,0,12.4779052734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,48,4,128,1,fp8,fp8,0,20.725497436523437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,48,8,128,1,float16,fp8,0,20.450587463378906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,48,8,128,1,fp8,fp8,0,20.854066467285158
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,48,8,128,1,float16,float16,0,24.662985229492186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,1,128,1,float16,fp8,0,10.066776275634766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,1,128,1,fp8,fp8,0,9.878791809082031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,2,128,1,float16,float16,0,12.86881103515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,2,128,1,float16,fp8,0,10.047652435302734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,2,128,1,fp8,fp8,0,10.009083557128907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,4,128,1,float16,fp8,0,10.344963073730469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,4,128,1,float16,float16,0,12.324201965332032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,4,128,1,fp8,fp8,0,10.054496002197265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,48,128,1,float16,float16,0,6.295512008666992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,48,128,1,float16,fp8,0,5.212319946289062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,8,128,1,float16,fp8,0,10.105233764648437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,8,128,1,float16,float16,0,12.466053009033203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,48,128,1,fp8,fp8,0,5.341886520385742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,48,8,128,1,fp8,fp8,0,10.167894744873047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,1,128,1,float16,fp8,0,4.949776077270508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,1,128,1,fp8,fp8,0,5.06396484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,1,128,1,float16,float16,0,6.157199859619141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,2,128,1,float16,fp8,0,4.9597423553466795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,2,128,1,float16,float16,0,6.007259368896484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,2,128,1,fp8,fp8,0,5.336153411865235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,4,128,1,float16,fp8,0,5.040760040283203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,4,128,1,float16,float16,0,6.277972793579101
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,4,128,1,fp8,fp8,0,5.083520126342774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,8,128,1,float16,fp8,0,5.129595184326172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,8,128,1,fp8,fp8,0,5.040983963012695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,48,128,1,float16,fp8,0,2.6627151489257814
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,48,8,128,1,float16,float16,0,5.9613903045654295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,48,128,1,fp8,fp8,0,2.6232063293457033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,1,128,1,float16,fp8,0,2.4953376770019533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,1,128,1,float16,float16,0,3.0207439422607423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,48,128,1,float16,float16,0,3.1983808517456054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,1,128,1,fp8,fp8,0,2.5015520095825194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,2,128,1,float16,fp8,0,2.526955223083496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,2,128,1,float16,float16,0,2.8234848022460937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,2,128,1,fp8,fp8,0,2.6369552612304688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,4,128,1,float16,float16,0,2.927177619934082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,4,128,1,float16,fp8,0,2.5161455154418944
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,48,128,1,float16,fp8,0,1.3057567596435546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,4,128,1,fp8,fp8,0,2.677254486083984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,48,128,1,float16,float16,0,1.7911231994628907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,8,128,1,float16,fp8,0,2.496559906005859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,8,128,1,float16,float16,0,2.7933071136474608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,48,128,1,fp8,fp8,0,1.4694416046142578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,1,128,1,float16,float16,0,1.3868608474731445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,1,128,1,float16,fp8,0,1.230780792236328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,1,128,1,fp8,fp8,0,1.2611007690429688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,2,128,1,float16,fp8,0,1.2582143783569335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,2,128,1,float16,float16,0,1.5273088455200194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,2,128,1,fp8,fp8,0,1.4659855842590332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,4,128,1,float16,float16,0,1.3924480438232423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,4,128,1,float16,fp8,0,1.4206975936889648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,4,128,1,fp8,fp8,0,1.2550864219665527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,8,128,1,float16,float16,0,1.3524592399597168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,8,128,1,float16,fp8,0,1.3877408027648925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,48,8,128,1,fp8,fp8,0,1.2885135650634765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,48,8,128,1,fp8,fp8,0,2.55053596496582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,48,1,128,1,float16,fp8,0,11.763057708740234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,48,1,128,1,fp8,fp8,0,11.882609558105468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,48,2,128,1,float16,fp8,0,11.42667236328125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,48,2,128,1,fp8,fp8,0,11.510470581054687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,48,1,128,1,float16,float16,0,14.48427734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,48,2,128,1,float16,float16,0,14.013020324707032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,48,4,128,1,float16,float16,0,13.874046325683594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,48,4,128,1,float16,fp8,0,11.549980926513673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,48,128,1,float16,fp8,0,6.357598495483399
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,48,128,1,float16,float16,0,7.569363403320312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,48,128,1,fp8,fp8,0,6.18596305847168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,48,4,128,1,fp8,fp8,0,11.672220611572266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,48,8,128,1,float16,fp8,0,11.741162872314453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,1,128,1,float16,float16,0,6.773593902587891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,48,8,128,1,fp8,fp8,0,11.673009490966797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,48,8,128,1,float16,float16,0,14.13779296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,1,128,1,float16,fp8,0,5.883548736572266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,1,128,1,fp8,fp8,0,5.970723342895508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,2,128,1,float16,float16,0,6.881617736816406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,2,128,1,float16,fp8,0,5.789471817016602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,2,128,1,fp8,fp8,0,5.718950271606445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,4,128,1,float16,fp8,0,5.912329483032226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,4,128,1,float16,float16,0,6.749460601806641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,4,128,1,fp8,fp8,0,5.954084777832032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,48,128,1,float16,float16,0,3.7254737854003905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,48,128,1,fp8,fp8,0,3.130598449707031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,8,128,1,float16,fp8,0,5.851166534423828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,8,128,1,float16,float16,0,7.068438720703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,1,128,1,float16,float16,0,3.3634033203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,48,8,128,1,fp8,fp8,0,6.179537582397461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,1,128,1,fp8,fp8,0,3.035411262512207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,48,128,1,float16,fp8,0,3.0718303680419923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,2,128,1,float16,fp8,0,2.8813791275024414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,2,128,1,float16,float16,0,3.5864032745361327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,2,128,1,fp8,fp8,0,3.098179244995117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,4,128,1,float16,float16,0,3.4604625701904297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,4,128,1,float16,fp8,0,2.999563217163086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,1,128,1,float16,fp8,0,2.903443145751953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,48,128,1,float16,float16,0,1.7529632568359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,4,128,1,fp8,fp8,0,2.8833423614501954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,8,128,1,float16,fp8,0,2.9308719635009766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,48,128,1,float16,fp8,0,1.8475919723510743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,8,128,1,float16,float16,0,3.1337055206298827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,48,128,1,fp8,fp8,0,1.5578144073486329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,48,8,128,1,fp8,fp8,0,3.185040092468262
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,1,128,1,float16,float16,0,1.5511167526245118
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,1,128,1,float16,fp8,0,1.5916895866394043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,2,128,1,float16,float16,0,1.5278176307678222
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,2,128,1,float16,fp8,0,1.621601676940918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,4,128,1,float16,float16,0,1.5953023910522461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,4,128,1,float16,fp8,0,1.4415247917175293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,4,128,1,fp8,fp8,0,1.553273582458496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,8,128,1,float16,float16,0,1.6023391723632812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,48,128,1,float16,float16,0,0.9467984199523926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,48,128,1,float16,fp8,0,0.7681248188018799
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,8,128,1,float16,fp8,0,1.49279842376709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,48,128,1,fp8,fp8,0,0.8158991813659668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,8,128,1,fp8,fp8,0,1.5344464302062988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,1,128,1,float16,float16,0,0.8384112358093262
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,1,128,1,float16,fp8,0,0.8154543876647949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,2,128,1,float16,float16,0,0.8097071647644043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,2,128,1,float16,fp8,0,0.734116792678833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,2,128,1,fp8,fp8,0,0.7393263816833496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,4,128,1,float16,float16,0,0.8508175849914551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,1,128,1,fp8,fp8,0,1.4465744018554687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,4,128,1,float16,fp8,0,0.7555391788482666
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,8,128,1,float16,float16,0,0.7842288017272949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,8,128,1,float16,fp8,0,0.7278719902038574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,48,2,128,1,fp8,fp8,0,1.5176447868347167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,8,128,1,fp8,fp8,0,0.8139439582824707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,1,128,1,fp8,fp8,0,0.7184288024902343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,48,4,128,1,fp8,fp8,0,0.8135359764099122
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,48,1,128,1,float16,fp8,0,10.85747833251953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,48,1,128,1,fp8,fp8,0,11.013998413085938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,48,2,128,1,float16,fp8,0,10.705257415771484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,48,2,128,1,fp8,fp8,0,10.839020538330079
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,48,1,128,1,float16,float16,0,12.460201263427734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,48,2,128,1,float16,float16,0,12.687067413330078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,48,4,128,1,float16,fp8,0,10.88695831298828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,48,4,128,1,float16,float16,0,12.669124603271484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,48,128,1,float16,fp8,0,5.872540664672852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,48,128,1,float16,float16,0,6.989952087402344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,48,128,1,fp8,fp8,0,5.921734237670899
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,1,128,1,float16,float16,0,6.22143669128418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,48,4,128,1,fp8,fp8,0,10.985609436035157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,48,8,128,1,float16,fp8,0,10.840206146240234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,48,8,128,1,fp8,fp8,0,11.096644592285156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,48,8,128,1,float16,float16,0,13.427072143554687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,1,128,1,float16,fp8,0,5.3310192108154295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,1,128,1,fp8,fp8,0,5.402431869506836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,2,128,1,float16,fp8,0,5.409344100952149
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,2,128,1,float16,float16,0,6.349233627319336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,2,128,1,fp8,fp8,0,5.583911895751953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,4,128,1,float16,fp8,0,5.57108154296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,4,128,1,float16,float16,0,6.240614318847657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,4,128,1,fp8,fp8,0,5.6492256164550785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,48,128,1,float16,float16,0,3.5946014404296873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,8,128,1,float16,float16,0,6.4528861999511715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,48,128,1,float16,fp8,0,2.9512847900390624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,48,128,1,fp8,fp8,0,2.9782447814941406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,8,128,1,float16,fp8,0,5.433660888671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,48,8,128,1,fp8,fp8,0,5.5070960998535154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,1,128,1,float16,fp8,0,2.6831071853637694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,1,128,1,fp8,fp8,0,2.846403121948242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,2,128,1,float16,fp8,0,2.7308399200439455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,2,128,1,fp8,fp8,0,2.772228813171387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,2,128,1,float16,float16,0,3.4377552032470704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,4,128,1,float16,fp8,0,2.7826911926269533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,4,128,1,float16,float16,0,3.411868667602539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,4,128,1,fp8,fp8,0,2.6824480056762696
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,48,128,1,float16,float16,0,1.7845375061035156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,1,128,1,float16,float16,0,3.145884895324707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,8,128,1,float16,fp8,0,2.763443183898926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,8,128,1,float16,float16,0,3.2512176513671873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,48,128,1,float16,fp8,0,1.5947584152221679
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,1,128,1,float16,float16,0,1.4282352447509765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,48,128,1,fp8,fp8,0,1.9295183181762696
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,1,128,1,float16,fp8,0,1.4521552085876466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,1,128,1,fp8,fp8,0,1.3317296028137207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,2,128,1,float16,float16,0,1.4544848442077636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,2,128,1,float16,fp8,0,1.4854191780090331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,2,128,1,fp8,fp8,0,1.478719997406006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,4,128,1,float16,float16,0,1.527340793609619
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,4,128,1,float16,fp8,0,1.397544002532959
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,4,128,1,fp8,fp8,0,1.3835503578186035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,48,128,1,float16,float16,0,0.8336607933044433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,8,128,1,float16,float16,0,1.4844304084777833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,48,128,1,fp8,fp8,0,0.7426415920257569
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,8,128,1,float16,fp8,0,1.457273578643799
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,48,8,128,1,fp8,fp8,0,1.370804786682129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,1,128,1,float16,float16,0,0.7102848052978515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,1,128,1,float16,fp8,0,0.739743995666504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,48,8,128,1,fp8,fp8,0,2.829096031188965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,1,128,1,fp8,fp8,0,0.7720640182495118
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,2,128,1,float16,fp8,0,0.721995210647583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,2,128,1,fp8,fp8,0,0.677126407623291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,4,128,1,float16,float16,0,0.7195663928985596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,4,128,1,float16,fp8,0,0.7267248153686523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,4,128,1,fp8,fp8,0,0.680518388748169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,8,128,1,float16,float16,0,0.8055343627929688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,48,128,1,float16,float16,0,0.42162718772888186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,8,128,1,fp8,fp8,0,0.6853616237640381
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,48,128,1,float16,fp8,0,0.38278400897979736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,48,128,1,fp8,fp8,0,0.37539360523223875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,1,128,1,float16,float16,0,0.38765439987182615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,1,128,1,float16,fp8,0,0.3608544111251831
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,2,128,1,float16,float16,0,0.3651439905166626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,2,128,1,fp8,fp8,0,0.3736079931259155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,4,128,1,float16,float16,0,0.3719808101654053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,4,128,1,float16,fp8,0,0.3475215911865234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,4,128,1,fp8,fp8,0,0.3699215888977051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,8,128,1,float16,float16,0,0.3783584117889404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,8,128,1,float16,fp8,0,0.3477952003479004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,48,128,1,float16,fp8,0,0.8011088371276855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,8,128,1,fp8,fp8,0,0.36212959289550783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,2,128,1,float16,float16,0,0.7152527809143067
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,48,8,128,1,float16,fp8,0,0.6955599784851074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,1,128,1,fp8,fp8,0,0.3430288076400757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,48,2,128,1,float16,fp8,0,0.34444479942321776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,48,1,128,1,float16,fp8,0,6.387192153930664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,48,1,128,1,fp8,fp8,0,6.324278259277344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,48,1,128,1,float16,float16,0,7.271510314941406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,48,2,128,1,float16,float16,0,7.443854522705078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,48,2,128,1,float16,fp8,0,6.345703887939453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,48,2,128,1,fp8,fp8,0,6.372280120849609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,48,4,128,1,float16,fp8,0,6.332420730590821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,48,4,128,1,float16,float16,0,7.438636779785156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,48,128,1,float16,float16,0,4.134648132324219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,48,128,1,float16,fp8,0,3.529000091552734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,48,4,128,1,fp8,fp8,0,6.4203651428222654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,48,128,1,fp8,fp8,0,3.509492874145508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,1,128,1,float16,float16,0,3.6687137603759767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,48,8,128,1,float16,fp8,0,6.504528045654297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,48,8,128,1,float16,float16,0,7.522425842285156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,48,8,128,1,fp8,fp8,0,6.548419189453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,1,128,1,float16,fp8,0,3.276286315917969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,1,128,1,fp8,fp8,0,3.149422454833984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,2,128,1,float16,fp8,0,3.167460823059082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,2,128,1,float16,float16,0,3.6528175354003904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,2,128,1,fp8,fp8,0,3.2646129608154295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,4,128,1,float16,float16,0,3.510609436035156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,4,128,1,float16,fp8,0,3.2572959899902343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,4,128,1,fp8,fp8,0,3.1819711685180665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,8,128,1,float16,fp8,0,3.216115188598633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,8,128,1,float16,float16,0,3.64107666015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,1,128,1,float16,float16,0,1.6767568588256836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,48,8,128,1,fp8,fp8,0,3.2135646820068358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,48,128,1,fp8,fp8,0,1.8478080749511718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,1,128,1,fp8,fp8,0,1.5835968017578126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,2,128,1,float16,float16,0,1.8087728500366211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,2,128,1,fp8,fp8,0,1.5885631561279296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,4,128,1,float16,float16,0,1.677560043334961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,48,128,1,float16,float16,0,2.121164894104004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,48,128,1,float16,fp8,0,1.7604095458984375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,4,128,1,float16,fp8,0,1.7720928192138672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,4,128,1,fp8,fp8,0,1.5957183837890625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,1,128,1,float16,fp8,0,1.5795295715332032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,8,128,1,float16,float16,0,1.8624576568603515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,48,128,1,float16,float16,0,0.98297119140625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,48,128,1,float16,fp8,0,0.8895152091979981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,48,128,1,fp8,fp8,0,0.883670425415039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,8,128,1,float16,fp8,0,1.6062335968017578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,8,128,1,fp8,fp8,0,1.6201663970947267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,1,128,1,float16,float16,0,0.949454402923584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,48,2,128,1,float16,fp8,0,1.6939920425415038
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,2,128,1,float16,float16,0,0.8429696083068847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,2,128,1,float16,fp8,0,0.7958015918731689
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,2,128,1,fp8,fp8,0,0.8032079696655273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,4,128,1,float16,float16,0,0.9243935585021973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,4,128,1,float16,fp8,0,0.7996096134185791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,4,128,1,fp8,fp8,0,0.8040736198425293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,8,128,1,float16,float16,0,0.8504032135009766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,8,128,1,float16,fp8,0,0.8088064193725586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,8,128,1,fp8,fp8,0,0.8940815925598145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,48,128,1,float16,float16,0,0.5029983997344971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,48,128,1,float16,fp8,0,0.44839839935302733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,48,128,1,fp8,fp8,0,0.4468688011169434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,1,128,1,float16,float16,0,0.42611041069030764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,1,128,1,float16,fp8,0,0.43087038993835447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,1,128,1,fp8,fp8,0,0.4128352165222168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,2,128,1,float16,float16,0,0.42797441482543946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,2,128,1,float16,fp8,0,0.4069215774536133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,2,128,1,fp8,fp8,0,0.40302238464355467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,4,128,1,float16,float16,0,0.435211181640625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,4,128,1,fp8,fp8,0,0.42206239700317383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,8,128,1,float16,float16,0,0.4308879852294922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,48,128,1,float16,float16,0,0.26737279891967775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,48,128,1,float16,fp8,0,0.23995680809020997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,48,128,1,fp8,fp8,0,0.23208320140838623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,1,128,1,float16,fp8,0,0.8208720207214355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,1,128,1,fp8,fp8,0,0.20877599716186523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,2,128,1,float16,float16,0,0.21924479007720948
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,2,128,1,float16,fp8,0,0.20926239490509033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,2,128,1,fp8,fp8,0,0.20863840579986573
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,4,128,1,float16,float16,0,0.2204927921295166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,4,128,1,float16,fp8,0,0.20942559242248535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,4,128,1,fp8,fp8,0,0.20975039005279542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,8,128,1,float16,float16,0,0.22452158927917482
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,8,128,1,float16,fp8,0,0.2114255905151367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,8,128,1,fp8,fp8,0,0.21242079734802247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,4,128,1,float16,fp8,0,0.41484799385070803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,8,128,1,float16,fp8,0,0.4093616008758545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,48,8,128,1,fp8,fp8,0,0.4242559909820557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,1,128,1,float16,fp8,0,0.21653919219970702
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,48,1,128,1,fp8,fp8,0,0.8001536369323731
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,48,1,128,1,float16,float16,0,0.22111680507659912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,48,1,128,1,float16,fp8,0,6.200611114501953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,48,1,128,1,float16,float16,0,6.986996459960937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,48,1,128,1,fp8,fp8,0,6.191870498657226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,48,2,128,1,float16,fp8,0,6.209222412109375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,48,2,128,1,float16,float16,0,7.1168670654296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,48,4,128,1,float16,float16,0,6.946206665039062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,48,2,128,1,fp8,fp8,0,6.209207916259766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,48,4,128,1,float16,fp8,0,6.2176048278808596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,48,4,128,1,fp8,fp8,0,6.192924880981446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,48,128,1,float16,fp8,0,3.603227233886719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,48,8,128,1,float16,fp8,0,6.28319206237793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,48,8,128,1,float16,float16,0,7.287417602539063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,48,8,128,1,fp8,fp8,0,6.283252716064453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,1,128,1,float16,float16,0,3.4236961364746095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,48,128,1,fp8,fp8,0,3.536830520629883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,1,128,1,float16,fp8,0,3.0696224212646483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,1,128,1,fp8,fp8,0,3.1111135482788086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,48,128,1,float16,float16,0,4.226768112182617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,2,128,1,float16,float16,0,3.200067138671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,2,128,1,fp8,fp8,0,3.0964656829833985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,2,128,1,float16,fp8,0,3.204006576538086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,4,128,1,float16,fp8,0,3.107054328918457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,4,128,1,float16,float16,0,3.571201705932617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,48,128,1,float16,fp8,0,1.7766799926757812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,8,128,1,float16,fp8,0,3.1503728866577148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,8,128,1,float16,float16,0,3.5359119415283202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,8,128,1,fp8,fp8,0,3.259902572631836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,48,128,1,fp8,fp8,0,1.7746000289916992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,1,128,1,float16,float16,0,1.7424720764160155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,1,128,1,float16,fp8,0,1.550499153137207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,1,128,1,fp8,fp8,0,1.5581744194030762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,2,128,1,float16,float16,0,1.7075824737548828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,2,128,1,float16,fp8,0,1.5615728378295899
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,2,128,1,fp8,fp8,0,1.559327983856201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,4,128,1,float16,fp8,0,1.5604096412658692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,4,128,1,fp8,fp8,0,1.569752025604248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,48,128,1,float16,float16,0,2.0041776657104493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,8,128,1,float16,float16,0,1.7356607437133789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,8,128,1,float16,fp8,0,1.5905952453613281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,48,128,1,float16,float16,0,1.0056303977966308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,48,128,1,float16,fp8,0,0.9006704330444336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,8,128,1,fp8,fp8,0,1.579843235015869
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,1,128,1,float16,float16,0,0.8691311836242676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,1,128,1,float16,fp8,0,0.7851136207580567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,1,128,1,fp8,fp8,0,0.7887695789337158
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,2,128,1,float16,float16,0,0.8166864395141602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,2,128,1,float16,fp8,0,0.7814752101898194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,2,128,1,fp8,fp8,0,0.7872288227081299
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,4,128,1,float16,float16,0,0.8201423645019531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,4,128,1,float16,fp8,0,0.8465904235839844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,4,128,1,fp8,fp8,0,0.7857456207275391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,48,4,128,1,fp8,fp8,0,3.0932079315185548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,48,4,128,1,float16,float16,0,1.7724319458007813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,8,128,1,float16,float16,0,0.8346943855285645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,8,128,1,fp8,fp8,0,0.8299008369445801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,48,128,1,float16,fp8,0,0.456056022644043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,48,128,1,fp8,fp8,0,0.45712480545043943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,1,128,1,float16,float16,0,0.4129327774047852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,1,128,1,float16,fp8,0,0.4000703811645508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,1,128,1,fp8,fp8,0,0.3962575912475586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,2,128,1,float16,float16,0,0.4137455940246582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,2,128,1,float16,fp8,0,0.41454238891601564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,4,128,1,float16,float16,0,0.41428318023681643
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,4,128,1,float16,fp8,0,0.4000671863555908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,4,128,1,fp8,fp8,0,0.41205439567565916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,8,128,1,float16,float16,0,0.42196002006530764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,8,128,1,float16,fp8,0,0.40593757629394533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,8,128,1,fp8,fp8,0,0.408135986328125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,48,128,1,float16,float16,0,0.2593456029891968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,48,128,1,float16,fp8,0,0.23531839847564698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,48,128,1,fp8,fp8,0,0.2366719961166382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,1,128,1,float16,float16,0,0.21077439785003663
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,1,128,1,float16,fp8,0,0.2034991979598999
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,1,128,1,fp8,fp8,0,0.20424959659576417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,2,128,1,float16,fp8,0,0.20490560531616211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,2,128,1,fp8,fp8,0,0.20543999671936036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,4,128,1,float16,float16,0,0.21522080898284912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,4,128,1,float16,fp8,0,0.2066175937652588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,8,128,1,float16,fp8,0,0.8387344360351563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,48,128,1,float16,float16,0,0.5055568218231201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,8,128,1,float16,float16,0,0.21911840438842772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,8,128,1,float16,fp8,0,0.2092303991317749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,8,128,1,fp8,fp8,0,0.20817759037017822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,48,128,1,float16,float16,0,0.13876960277557374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,48,128,1,float16,fp8,0,0.12475680112838745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,48,128,1,fp8,fp8,0,0.1241968035697937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,1,128,1,float16,float16,0,0.11115360260009766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,1,128,1,float16,fp8,0,0.10485119819641113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,1,128,1,fp8,fp8,0,0.10468480587005616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,2,128,1,float16,float16,0,0.11181119680404664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,2,128,1,fp8,fp8,0,0.10534720420837403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,4,128,1,float16,float16,0,0.11326400041580201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,4,128,1,float16,fp8,0,0.10627679824829102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,4,128,1,fp8,fp8,0,0.1070255994796753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,8,128,1,float16,float16,0,0.11438720226287842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,8,128,1,float16,fp8,0,0.10921119451522827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,8,128,1,fp8,fp8,0,0.10823680162429809
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,2,128,1,float16,float16,0,0.21184320449829103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,48,4,128,1,fp8,fp8,0,0.2055840015411377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,48,2,128,1,float16,fp8,0,0.1053663969039917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,48,2,128,1,fp8,fp8,0,0.3994719982147217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,48,48,128,1,fp8,fp8,0,0.8962688446044922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,48,1,128,1,fp8,fp8,0,3.8086479187011717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,48,1,128,1,float16,float16,0,3.9552478790283203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,48,2,128,1,float16,float16,0,4.111135864257813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,48,2,128,1,float16,fp8,0,3.8225936889648438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,48,2,128,1,fp8,fp8,0,3.845393753051758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,48,4,128,1,float16,float16,0,4.066457748413086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,48,4,128,1,float16,fp8,0,3.845830535888672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,48,1,128,1,float16,fp8,0,3.826686477661133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,48,4,128,1,fp8,fp8,0,3.8188144683837892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,48,8,128,1,float16,float16,0,4.3200641632080075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,48,128,1,float16,float16,0,2.5264463424682617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,1,128,1,float16,float16,0,1.9579744338989258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,48,128,1,fp8,fp8,0,2.248676872253418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,48,8,128,1,fp8,fp8,0,3.871150588989258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,1,128,1,float16,fp8,0,1.9097312927246093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,1,128,1,fp8,fp8,0,2.025864028930664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,2,128,1,float16,float16,0,2.0961503982543945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,2,128,1,fp8,fp8,0,1.9099039077758788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,4,128,1,float16,fp8,0,1.923624038696289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,4,128,1,fp8,fp8,0,1.939156723022461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,48,8,128,1,float16,fp8,0,3.9745697021484374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,48,128,1,float16,fp8,0,2.350720024108887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,8,128,1,float16,fp8,0,1.9514127731323243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,8,128,1,float16,float16,0,2.150217628479004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,8,128,1,fp8,fp8,0,1.951531219482422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,48,128,1,float16,fp8,0,1.1519344329833985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,48,128,1,fp8,fp8,0,1.149137592315674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,1,128,1,float16,float16,0,0.9791775703430176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,1,128,1,fp8,fp8,0,0.9592320442199707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,2,128,1,float16,fp8,0,1.981999969482422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,48,4,128,1,float16,float16,0,2.0274127960205077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,2,128,1,float16,fp8,0,0.9740176200866699
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,2,128,1,fp8,fp8,0,0.9587488174438477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,4,128,1,float16,float16,0,1.0069600105285645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,4,128,1,float16,fp8,0,0.9776144027709961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,4,128,1,fp8,fp8,0,0.969206428527832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,8,128,1,float16,float16,0,1.03438081741333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,8,128,1,float16,fp8,0,0.979958438873291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,48,128,1,float16,float16,0,0.6372623920440674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,8,128,1,fp8,fp8,0,1.0125087738037108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,48,128,1,float16,float16,0,1.2518447875976562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,48,128,1,fp8,fp8,0,0.581279993057251
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,1,128,1,float16,float16,0,0.4958640098571777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,1,128,1,float16,fp8,0,0.49164161682128904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,1,128,1,fp8,fp8,0,0.49296159744262696
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,2,128,1,float16,float16,0,0.5048416137695313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,2,128,1,float16,fp8,0,0.48600478172302247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,2,128,1,fp8,fp8,0,0.492460823059082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,4,128,1,float16,float16,0,0.5082528114318847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,1,128,1,float16,fp8,0,1.0452832221984862
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,4,128,1,float16,fp8,0,0.490828800201416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,4,128,1,fp8,fp8,0,0.4891024112701416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,48,2,128,1,float16,float16,0,0.9969327926635743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,8,128,1,float16,float16,0,0.5198095798492431
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,8,128,1,float16,fp8,0,0.49934401512146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,48,128,1,float16,float16,0,0.32876479625701904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,8,128,1,fp8,fp8,0,0.503436803817749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,48,128,1,float16,fp8,0,0.2953871965408325
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,48,128,1,fp8,fp8,0,0.29669439792633057
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,1,128,1,float16,float16,0,0.2564239978790283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,1,128,1,float16,fp8,0,0.2497328042984009
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,1,128,1,fp8,fp8,0,0.24944639205932617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,2,128,1,float16,fp8,0,0.25031518936157227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,2,128,1,fp8,fp8,0,0.2504688024520874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,4,128,1,float16,float16,0,0.2609679937362671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,4,128,1,float16,fp8,0,0.25176000595092773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,4,128,1,fp8,fp8,0,0.2523695945739746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,8,128,1,float16,float16,0,0.26437599658966066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,8,128,1,float16,fp8,0,0.2566911935806274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,8,128,1,fp8,fp8,0,0.25924479961395264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,48,128,1,float16,float16,0,0.17125920057296753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,48,128,1,fp8,fp8,0,0.1559391975402832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,1,128,1,float16,float16,0,0.1340559959411621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,1,128,1,float16,fp8,0,0.1324944019317627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,1,128,1,fp8,fp8,0,0.13085440397262574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,2,128,1,float16,float16,0,0.13401600122451782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,2,128,1,float16,fp8,0,0.13203359842300416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,2,128,1,fp8,fp8,0,0.13314399719238282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,4,128,1,float16,float16,0,0.13600319623947144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,4,128,1,float16,fp8,0,0.13202879428863526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,4,128,1,fp8,fp8,0,0.13211359977722167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,8,128,1,float16,float16,0,0.1405295968055725
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,48,128,1,float16,float16,0,0.09126880168914794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,48,128,1,float16,fp8,0,0.08604639768600464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,48,128,1,fp8,fp8,0,0.08527839779853821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,1,128,1,float16,float16,0,0.07070559859275818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,1,128,1,float16,fp8,0,0.06846719980239868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,1,128,1,fp8,fp8,0,0.06897280216217042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,2,128,1,float16,fp8,0,0.06890079975128174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,2,128,1,fp8,fp8,0,0.06892799735069274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,4,128,1,float16,float16,0,0.0721552014350891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,4,128,1,float16,fp8,0,0.06860960125923157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,4,128,1,fp8,fp8,0,0.06881120204925537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,8,128,1,float16,float16,0,0.07608799934387207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,8,128,1,fp8,fp8,0,0.07004960179328919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,8,128,1,float16,fp8,0,0.07010719776153565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,48,2,128,1,float16,float16,0,0.2576864004135132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,48,128,1,float16,fp8,0,0.15634239912033082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,48,48,128,1,float16,fp8,0,0.5784704208374023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,8,128,1,float16,fp8,0,0.13441280126571656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,48,2,128,1,float16,float16,0,0.07112159729003906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,48,1,128,1,float16,float16,0,4.155612945556641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,48,1,128,1,float16,fp8,0,3.985707092285156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,48,1,128,1,fp8,fp8,0,4.013768005371094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,48,2,128,1,float16,float16,0,4.2658641815185545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,48,8,128,1,fp8,fp8,0,0.13388320207595825
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,48,2,128,1,float16,fp8,0,3.987518310546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,48,2,128,1,fp8,fp8,0,3.9925743103027345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,48,4,128,1,float16,float16,0,4.211313629150391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,48,4,128,1,float16,fp8,0,4.064308929443359
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,48,128,1,float16,fp8,0,2.4483232498168945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,48,128,1,float16,float16,0,2.7927087783813476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,48,8,128,1,float16,float16,0,4.460416030883789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,48,128,1,fp8,fp8,0,2.495631980895996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,48,8,128,1,fp8,fp8,0,4.100803375244141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,48,8,128,1,float16,fp8,0,4.1318817138671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,1,128,1,float16,float16,0,2.037766456604004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,1,128,1,float16,fp8,0,1.9911535263061524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,1,128,1,fp8,fp8,0,2.0804672241210938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,2,128,1,float16,float16,0,2.0914703369140626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,2,128,1,float16,fp8,0,2.0153631210327148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,2,128,1,fp8,fp8,0,1.9928815841674805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,4,128,1,float16,float16,0,2.1039072036743165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,4,128,1,float16,fp8,0,2.0880096435546873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,4,128,1,fp8,fp8,0,2.0068639755249023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,48,128,1,float16,float16,0,1.3731295585632324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,48,128,1,float16,fp8,0,1.3166895866394044
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,8,128,1,float16,fp8,0,2.1246368408203127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,8,128,1,fp8,fp8,0,2.0700704574584963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,1,128,1,float16,float16,0,1.0014944076538086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,48,128,1,fp8,fp8,0,1.2514063835144043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,1,128,1,float16,fp8,0,1.015675163269043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,1,128,1,fp8,fp8,0,1.03580961227417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,2,128,1,float16,float16,0,1.015193557739258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,2,128,1,float16,fp8,0,1.0155759811401368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,2,128,1,fp8,fp8,0,1.0009008407592774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,4,128,1,float16,float16,0,1.033390426635742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,48,4,128,1,fp8,fp8,0,4.0203296661376955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,4,128,1,float16,fp8,0,1.0249839782714845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,4,128,1,fp8,fp8,0,1.021998405456543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,8,128,1,float16,float16,0,1.0825632095336915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,8,128,1,float16,fp8,0,1.0442288398742676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,48,128,1,float16,float16,0,0.7049712181091309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,48,128,1,float16,fp8,0,0.6251423835754395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,1,128,1,float16,float16,0,0.5120495796203614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,48,128,1,fp8,fp8,0,0.6292975902557373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,1,128,1,float16,fp8,0,0.5119440078735351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,48,8,128,1,float16,float16,0,2.1528528213500975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,1,128,1,fp8,fp8,0,0.5128511905670166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,2,128,1,float16,fp8,0,0.5117536067962647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,2,128,1,fp8,fp8,0,0.5115935802459717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,4,128,1,float16,float16,0,0.5239503860473633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,4,128,1,fp8,fp8,0,0.5166639804840087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,8,128,1,float16,float16,0,0.539795207977295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,8,128,1,float16,fp8,0,0.5263423919677734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,8,128,1,fp8,fp8,0,0.5294960021972657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,48,128,1,float16,float16,0,0.35785760879516604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,48,128,1,float16,fp8,0,0.3215264081954956
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,48,128,1,fp8,fp8,0,0.3214735984802246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,1,128,1,float16,float16,0,0.26257920265197754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,1,128,1,float16,fp8,0,0.2614383935928345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,2,128,1,float16,float16,0,0.26282880306243894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,2,128,1,float16,fp8,0,0.2617552042007446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,2,128,1,fp8,fp8,0,0.26220319271087644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,4,128,1,float16,float16,0,0.2675152063369751
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,4,128,1,float16,fp8,0,0.2632927894592285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,4,128,1,fp8,fp8,0,0.2639904022216797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,8,128,1,float16,float16,0,0.27556960582733153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,8,128,1,float16,fp8,0,0.2693279981613159
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,8,128,1,fp8,fp8,0,0.26986720561981203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,48,128,1,float16,float16,0,0.18612799644470215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,48,128,1,float16,fp8,0,0.16737120151519774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,48,8,128,1,fp8,fp8,0,1.028724765777588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,48,128,1,fp8,fp8,0,0.16773439645767213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,1,128,1,float16,float16,0,0.13691680431365966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,1,128,1,float16,fp8,0,0.13573119640350342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,1,128,1,fp8,fp8,0,0.13400160074234008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,2,128,1,float16,fp8,0,0.13650879859924317
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,2,128,1,fp8,fp8,0,0.13644319772720337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,4,128,1,float16,float16,0,0.13897440433502198
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,2,128,1,float16,float16,0,0.5105279922485352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,4,128,1,fp8,fp8,0,0.13767999410629272
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,8,128,1,float16,float16,0,0.143121600151062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,8,128,1,float16,fp8,0,0.13934240341186524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,8,128,1,fp8,fp8,0,0.14007680416107177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,48,128,1,float16,fp8,0,0.08987039923667908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,48,128,1,fp8,fp8,0,0.09024320244789123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,1,128,1,float16,float16,0,0.07359200119972228
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,1,128,1,float16,fp8,0,0.07048799991607665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,1,128,1,fp8,fp8,0,0.06994079947471618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,2,128,1,float16,float16,0,0.07449439764022828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,2,128,1,float16,fp8,0,0.07024480104446411
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,2,128,1,fp8,fp8,0,0.07089920043945312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,4,128,1,float16,float16,0,0.07544959783554077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,48,1,128,1,fp8,fp8,0,0.2608911991119385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,4,128,1,fp8,fp8,0,0.07161759734153747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,8,128,1,float16,float16,0,0.07814239859580993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,8,128,1,float16,fp8,0,0.07374560236930847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,8,128,1,fp8,fp8,0,0.07411199808120728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,48,128,1,float16,float16,0,0.058499199151992795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,48,128,1,float16,fp8,0,0.051900798082351686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,1,128,1,float16,float16,0,0.03974240124225616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,1,128,1,float16,fp8,0,0.03908160030841827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,1,128,1,fp8,fp8,0,0.03873760104179382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,2,128,1,float16,float16,0,0.04017919898033142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,2,128,1,fp8,fp8,0,0.03909760117530823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,4,128,1,float16,float16,0,0.040214401483535764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,4,128,1,float16,fp8,0,0.039124798774719236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,4,128,1,fp8,fp8,0,0.03905119895935059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,8,128,1,float16,float16,0,0.04090720117092132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,2,128,1,float16,float16,0,0.13850560188293456
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,8,128,1,float16,fp8,0,0.0398496001958847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,8,128,1,fp8,fp8,0,0.03973279893398285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,48,4,128,1,float16,fp8,0,0.13759360313415528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,48,128,1,float16,float16,0,0.09952639937400817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,48,4,128,1,float16,fp8,0,0.5165279865264892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,48,4,128,1,float16,fp8,0,0.07185440063476563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,48,128,1,fp8,fp8,0,0.05166400074958801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,48,2,128,1,float16,fp8,0,0.038771200180053714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,48,1,128,1,float16,float16,0,2.786612892150879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,48,1,128,1,float16,fp8,0,2.8846927642822267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,48,1,128,1,fp8,fp8,0,2.866640090942383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,48,2,128,1,float16,float16,0,2.8158416748046875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,48,2,128,1,float16,fp8,0,2.8734415054321287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,48,2,128,1,fp8,fp8,0,2.872488021850586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,48,4,128,1,float16,float16,0,2.8660240173339844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,48,4,128,1,float16,fp8,0,2.8986543655395507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,48,4,128,1,fp8,fp8,0,2.916201591491699
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,48,128,1,float16,float16,0,2.141401672363281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,48,8,128,1,float16,fp8,0,2.979840087890625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,48,8,128,1,fp8,fp8,0,2.9935136795043946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,48,128,1,float16,fp8,0,1.95239200592041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,1,128,1,float16,float16,0,1.3900416374206543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,48,128,1,fp8,fp8,0,1.983990478515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,1,128,1,float16,fp8,0,1.4449024200439453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,1,128,1,fp8,fp8,0,1.427337646484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,2,128,1,float16,float16,0,1.3951104164123536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,2,128,1,float16,fp8,0,1.4444751739501953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,2,128,1,fp8,fp8,0,1.4441951751708983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,4,128,1,float16,float16,0,1.43023681640625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,4,128,1,float16,fp8,0,1.4567392349243165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,4,128,1,fp8,fp8,0,1.4688752174377442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,8,128,1,float16,float16,0,1.4860112190246582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,48,8,128,1,float16,float16,0,2.992867279052734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,8,128,1,fp8,fp8,0,1.502190399169922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,48,128,1,float16,float16,0,1.081974411010742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,48,128,1,fp8,fp8,0,0.9904159545898438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,1,128,1,float16,fp8,0,0.7327727794647216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,1,128,1,fp8,fp8,0,0.7265632152557373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,2,128,1,float16,float16,0,0.7067791938781738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,2,128,1,float16,fp8,0,0.7344480037689209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,2,128,1,fp8,fp8,0,0.7342448234558105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,4,128,1,float16,float16,0,0.7246047973632812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,4,128,1,float16,fp8,0,0.7340271949768067
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,4,128,1,fp8,fp8,0,0.7393072128295899
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,8,128,1,float16,float16,0,0.7604496002197265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,8,128,1,float16,fp8,0,0.7604959964752197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,8,128,1,fp8,fp8,0,0.7574495792388916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,48,128,1,float16,float16,0,0.5504879951477051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,48,128,1,float16,fp8,0,0.5015952110290527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,48,8,128,1,float16,fp8,0,1.5104880332946777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,48,128,1,fp8,fp8,0,0.5004191875457764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,1,128,1,float16,float16,0,0.35745599269866946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,1,128,1,float16,fp8,0,0.37112960815429685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,48,128,1,float16,fp8,0,0.9832991600036621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,1,128,1,fp8,fp8,0,0.3718480110168457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,2,128,1,float16,float16,0,0.3634079933166504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,2,128,1,float16,fp8,0,0.3711199998855591
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,4,128,1,float16,float16,0,0.3702064037322998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,4,128,1,float16,fp8,0,0.374398398399353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,8,128,1,float16,float16,0,0.3869407892227173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,8,128,1,float16,fp8,0,0.3876159906387329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,48,128,1,float16,float16,0,0.28245599269866944
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,48,128,1,float16,fp8,0,0.2575472116470337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,48,128,1,fp8,fp8,0,0.2554896116256714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,1,128,1,float16,float16,0,0.1859328031539917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,1,128,1,float16,fp8,0,0.1918223977088928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,1,128,1,fp8,fp8,0,0.19003839492797853
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,2,128,1,float16,fp8,0,0.19294240474700927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,2,128,1,fp8,fp8,0,0.19061280488967897
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,4,128,1,float16,float16,0,0.19258079528808594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,4,128,1,float16,fp8,0,0.19225599765777587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,4,128,1,fp8,fp8,0,0.19446239471435547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,8,128,1,float16,float16,0,0.19911199808120728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,8,128,1,float16,fp8,0,0.20096321105957032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,8,128,1,fp8,fp8,0,0.19803199768066407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,48,128,1,float16,float16,0,0.1489359974861145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,48,128,1,float16,fp8,0,0.13392640352249147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,48,128,1,fp8,fp8,0,0.135043203830719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,1,128,1,float16,fp8,0,0.10028480291366577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,1,128,1,fp8,fp8,0,0.10038880109786988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,2,128,1,float16,float16,0,0.09976159930229186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,2,128,1,float16,fp8,0,0.1010159969329834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,2,128,1,fp8,fp8,0,0.10147839784622192
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,48,1,128,1,float16,float16,0,0.7179664134979248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,4,128,1,float16,float16,0,0.10118240118026733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,4,128,1,fp8,fp8,0,0.3778543949127197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,4,128,1,fp8,fp8,0,0.10339679718017578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,8,128,1,float16,float16,0,0.1059391975402832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,8,128,1,fp8,fp8,0,0.38392479419708253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,8,128,1,fp8,fp8,0,0.10659359693527222
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,48,128,1,float16,float16,0,0.08089280128479004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,48,128,1,float16,fp8,0,0.07371360063552856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,1,128,1,float16,float16,0,0.055294400453567503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,1,128,1,float16,fp8,0,0.0537663996219635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,1,128,1,fp8,fp8,0,0.053750401735305785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,2,128,1,float16,float16,0,0.05617920160293579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,2,128,1,float16,fp8,0,0.05442399978637695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,48,2,128,1,float16,float16,0,0.18733439445495606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,4,128,1,float16,float16,0,0.057543998956680296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,4,128,1,float16,fp8,0,0.05578399896621704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,4,128,1,fp8,fp8,0,0.05481280088424682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,8,128,1,float16,float16,0,0.059464001655578615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,8,128,1,float16,fp8,0,0.057715201377868654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,8,128,1,fp8,fp8,0,0.05845440030097961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,48,128,1,float16,float16,0,0.04706720113754272
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,48,128,1,fp8,fp8,0,0.04173440039157868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,1,128,1,float16,float16,0,0.03125759959220886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,1,128,1,float16,fp8,0,0.031841599941253663
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,1,128,1,fp8,fp8,0,0.03215200006961823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,2,128,1,float16,float16,0,0.03172959983348846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,2,128,1,float16,fp8,0,0.03205919861793518
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,2,128,1,fp8,fp8,0,0.03200959861278534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,1,128,1,float16,float16,0,0.10027999877929687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,4,128,1,float16,float16,0,0.032153600454330446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,4,128,1,float16,fp8,0,0.03223839998245239
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,4,128,1,fp8,fp8,0,0.031999999284744264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,8,128,1,float16,float16,0,0.03264960050582886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,8,128,1,float16,fp8,0,0.03291040062904358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,8,128,1,fp8,fp8,0,0.033188799023628236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,48,2,128,1,fp8,fp8,0,0.36955039501190184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,48,128,1,float16,fp8,0,0.025824001431465148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,48,128,1,fp8,fp8,0,0.025867199897766112
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,4,128,1,float16,fp8,0,0.10204800367355346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,1,128,1,float16,fp8,0,0.020295999944210052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,1,128,1,fp8,fp8,0,0.020392000675201416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,2,128,1,float16,float16,0,0.02096319943666458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,2,128,1,float16,fp8,0,0.020505599677562714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,48,8,128,1,float16,fp8,0,0.1075279951095581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,4,128,1,float16,float16,0,0.021142399311065672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,4,128,1,float16,fp8,0,0.020457600057125092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,48,128,1,fp8,fp8,0,0.07381280064582825
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,8,128,1,float16,float16,0,0.02140959948301315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,8,128,1,float16,fp8,0,0.02253440022468567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,8,128,1,fp8,fp8,0,0.02252800017595291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,48,48,128,1,float16,fp8,0,0.041982400417327884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,48,1,128,1,float16,float16,0,1.0905535697937012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,48,1,128,1,float16,fp8,0,1.1736623764038085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,48,128,1,float16,float16,0,0.026811200380325317
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,1,128,1,float16,float16,0,0.02065120041370392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,48,1,128,1,fp8,fp8,0,1.16495361328125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,2,128,1,fp8,fp8,0,0.0203247994184494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,48,4,128,1,fp8,fp8,0,0.020401600003242492
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,48,2,128,1,float16,float16,0,1.1088751792907714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,48,2,128,1,fp8,fp8,0,0.054313600063323975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,48,2,128,1,float16,fp8,0,1.169092845916748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,48,2,128,1,fp8,fp8,0,1.1777536392211914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,48,4,128,1,float16,float16,0,1.13297119140625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,48,4,128,1,float16,fp8,0,1.1886544227600098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,48,4,128,1,fp8,fp8,0,1.1831567764282227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,48,8,128,1,float16,float16,0,1.2029007911682128
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,48,8,128,1,float16,fp8,0,1.2259535789489746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,48,128,1,float16,fp8,0,0.81080322265625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,48,128,1,float16,float16,0,0.9500991821289062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,48,8,128,1,fp8,fp8,0,1.2255616188049316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,1,128,1,float16,float16,0,0.552953577041626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,48,128,1,fp8,fp8,0,0.8146847724914551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,1,128,1,float16,fp8,0,0.5928304195404053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,2,128,1,float16,float16,0,0.5579599857330322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,2,128,1,float16,fp8,0,0.5971648216247558
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,4,128,1,float16,float16,0,0.5738383769989014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,4,128,1,float16,fp8,0,0.602623987197876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,4,128,1,fp8,fp8,0,0.6028687953948975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,8,128,1,float16,float16,0,0.6051743984222412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,8,128,1,float16,fp8,0,0.6263792037963867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,48,128,1,float16,float16,0,0.48256797790527345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,8,128,1,fp8,fp8,0,0.6215087890625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,48,128,1,float16,fp8,0,0.4151519775390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,48,128,1,fp8,fp8,0,0.41260480880737305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,1,128,1,float16,float16,0,0.2856816053390503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,1,128,1,float16,fp8,0,0.30104639530181887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,1,128,1,fp8,fp8,0,0.30330719947814944
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,2,128,1,float16,fp8,0,0.3043839931488037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,2,128,1,fp8,fp8,0,0.3033423900604248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,4,128,1,float16,float16,0,0.29650559425354006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,4,128,1,float16,fp8,0,0.3062207937240601
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,8,128,1,float16,float16,0,0.312011194229126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,8,128,1,float16,fp8,0,0.3176464080810547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,8,128,1,fp8,fp8,0,0.3200576066970825
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,48,128,1,float16,float16,0,0.24908480644226075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,48,128,1,float16,fp8,0,0.2148207902908325
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,48,128,1,fp8,fp8,0,0.21403839588165283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,1,128,1,float16,float16,0,0.14966399669647218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,1,128,1,float16,fp8,0,0.15660320520401
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,1,128,1,fp8,fp8,0,0.1577888011932373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,2,128,1,float16,float16,0,0.1507807970046997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,2,128,1,float16,fp8,0,0.15904799699783326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,2,128,1,fp8,fp8,0,0.1571087956428528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,4,128,1,float16,fp8,0,0.15995359420776367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,8,128,1,float16,float16,0,0.16228480339050294
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,8,128,1,float16,fp8,0,0.16589440107345582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,1,128,1,fp8,fp8,0,0.5895919799804688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,8,128,1,fp8,fp8,0,0.1657088041305542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,48,128,1,float16,float16,0,0.13178720474243164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,48,2,128,1,fp8,fp8,0,0.5900800228118896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,48,128,1,float16,fp8,0,0.11375999450683594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,1,128,1,float16,float16,0,0.08063520193099975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,1,128,1,float16,fp8,0,0.08475040197372437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,1,128,1,fp8,fp8,0,0.0841759979724884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,2,128,1,float16,float16,0,0.08266080021858216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,2,128,1,float16,fp8,0,0.08530719876289368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,2,128,1,fp8,fp8,0,0.08483359813690186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,4,128,1,float16,float16,0,0.08459360003471375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,4,128,1,float16,fp8,0,0.08621280193328858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,4,128,1,fp8,fp8,0,0.08591840267181397
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,8,128,1,float16,float16,0,0.08853600025177003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,8,128,1,float16,fp8,0,0.08893600106239319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,8,128,1,fp8,fp8,0,0.08889439702033997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,48,128,1,float16,float16,0,0.07372320294380189
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,48,128,1,float16,fp8,0,0.0639631986618042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,48,128,1,fp8,fp8,0,0.06366400122642517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,1,128,1,float16,float16,0,0.046291199326515195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,1,128,1,float16,fp8,0,0.0455487996339798
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,1,128,1,fp8,fp8,0,0.04541119933128357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,2,128,1,float16,float16,0,0.04710719883441925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,2,128,1,fp8,fp8,0,0.045510399341583255
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,2,128,1,float16,float16,0,0.2864703893661499
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,4,128,1,float16,float16,0,0.048721599578857425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,4,128,1,fp8,fp8,0,0.046409600973129274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,8,128,1,float16,float16,0,0.050040000677108766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,8,128,1,float16,fp8,0,0.04963679909706116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,8,128,1,fp8,fp8,0,0.04967359900474548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,48,128,1,float16,float16,0,0.04216960072517395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,48,128,1,float16,fp8,0,0.03646720051765442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,48,128,1,fp8,fp8,0,0.03649120032787323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,1,128,1,float16,float16,0,0.02547680139541626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,1,128,1,float16,fp8,0,0.027353599667549133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,1,128,1,fp8,fp8,0,0.02756800055503845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,2,128,1,float16,float16,0,0.02555040121078491
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,2,128,1,float16,fp8,0,0.02746720016002655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,2,128,1,fp8,fp8,0,0.027342399954795836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,4,128,1,float16,float16,0,0.02598080039024353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,4,128,1,float16,fp8,0,0.027422401309013366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,4,128,1,fp8,fp8,0,0.027507200837135315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,4,128,1,float16,float16,0,0.1555855989456177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,48,4,128,1,fp8,fp8,0,0.16084640026092528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,8,128,1,fp8,fp8,0,0.02789120078086853
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,48,128,1,float16,float16,0,0.02478879988193512
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,48,128,1,float16,fp8,0,0.022281600534915923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,48,128,1,fp8,fp8,0,0.02223840057849884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,1,128,1,float16,float16,0,0.019012799859046935
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,1,128,1,float16,fp8,0,0.018612800538539885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,1,128,1,fp8,fp8,0,0.018643200397491455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,2,128,1,float16,float16,0,0.019284799695014954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,2,128,1,float16,fp8,0,0.018697600066661834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,2,128,1,fp8,fp8,0,0.018676799535751343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,4,128,1,float16,float16,0,0.01934400051832199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,4,128,1,float16,fp8,0,0.01852000057697296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,4,128,1,fp8,fp8,0,0.01852640062570572
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,8,128,1,float16,float16,0,0.019657599925994872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,2,128,1,float16,fp8,0,0.046089598536491395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,8,128,1,fp8,fp8,0,0.020396800339221956
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,48,128,1,float16,float16,0,0.016476799547672272
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,48,128,1,float16,fp8,0,0.016385599970817566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,48,4,128,1,fp8,fp8,0,0.30846080780029295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,48,128,1,fp8,fp8,0,0.016164800524711607
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,1,128,1,float16,float16,0,0.014388799667358398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,1,128,1,fp8,fp8,0,0.014867199957370758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,2,128,1,float16,float16,0,0.014433600008487701
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,2,128,1,float16,fp8,0,0.015123200416564942
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,2,128,1,fp8,fp8,0,0.015143999457359314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,4,128,1,float16,float16,0,0.014585599303245544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,4,128,1,float16,fp8,0,0.015321600437164306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,8,128,1,float16,float16,0,0.026662400364875792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,48,8,128,1,float16,fp8,0,0.027939200401306152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,8,128,1,float16,fp8,0,0.015294399857521058
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,8,128,1,fp8,fp8,0,0.015521599352359772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,48,48,128,1,fp8,fp8,0,0.11396000385284424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,48,1,128,1,float16,float16,0,0.5198368072509766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,48,1,128,1,float16,fp8,0,0.5608831882476807
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,48,1,128,1,fp8,fp8,0,0.5617311954498291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,48,8,128,1,float16,fp8,0,0.020712000131607056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,48,4,128,1,float16,fp8,0,0.04609439969062805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,48,2,128,1,float16,float16,0,0.523196792602539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,48,2,128,1,float16,fp8,0,0.5653215885162354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,1,128,1,float16,fp8,0,0.01494079977273941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,48,2,128,1,fp8,fp8,0,0.5657360076904296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,48,4,128,1,float16,float16,0,0.5384624004364014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,48,4,128,1,float16,fp8,0,0.5773663997650147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,48,4,128,1,fp8,fp8,0,0.5773791790008544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,48,8,128,1,float16,float16,0,0.5660496234893799
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,4,128,1,fp8,fp8,0,0.01510239988565445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,48,128,1,float16,float16,0,0.459768009185791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,48,8,128,1,float16,float16,0,0.01478399932384491
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,48,128,1,float16,fp8,0,0.4135312080383301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,1,128,1,float16,float16,0,0.26723198890686034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,1,128,1,float16,fp8,0,0.2884191989898682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,2,128,1,float16,float16,0,0.26790239810943606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,2,128,1,float16,fp8,0,0.2847424030303955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,4,128,1,float16,float16,0,0.27677919864654543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,4,128,1,float16,fp8,0,0.2945183992385864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,4,128,1,fp8,fp8,0,0.2940351963043213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,8,128,1,float16,float16,0,0.29012479782104494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,8,128,1,float16,fp8,0,0.3061552047729492
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,8,128,1,fp8,fp8,0,0.306715202331543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,48,128,1,float16,fp8,0,0.21388320922851561
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,1,128,1,float16,float16,0,0.14112639427185059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,1,128,1,float16,fp8,0,0.15048320293426515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,1,128,1,fp8,fp8,0,0.15108319520950317
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,2,128,1,float16,float16,0,0.1422816038131714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,2,128,1,float16,fp8,0,0.1527343988418579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,2,128,1,fp8,fp8,0,0.14969600439071656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,48,8,128,1,fp8,fp8,0,0.5990015983581543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,4,128,1,float16,float16,0,0.14510879516601563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,48,128,1,fp8,fp8,0,0.41353597640991213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,4,128,1,fp8,fp8,0,0.15395040512084962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,1,128,1,fp8,fp8,0,0.2868688106536865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,8,128,1,float16,float16,0,0.15167679786682128
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,8,128,1,float16,fp8,0,0.15994080305099487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,8,128,1,fp8,fp8,0,0.16027679443359374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,48,128,1,float16,float16,0,0.12647520303726195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,48,128,1,fp8,fp8,0,0.11246720552444459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,1,128,1,float16,float16,0,0.07702239751815795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,1,128,1,float16,fp8,0,0.0804032027721405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,1,128,1,fp8,fp8,0,0.08062559962272645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,2,128,1,float16,float16,0,0.07829920053482056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,2,128,1,float16,fp8,0,0.08161119818687439
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,2,128,1,fp8,fp8,0,0.08128640055656433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,48,128,1,float16,float16,0,0.23644640445709228
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,4,128,1,float16,fp8,0,0.08262079954147339
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,4,128,1,fp8,fp8,0,0.0825984001159668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,8,128,1,float16,float16,0,0.08318719863891602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,48,128,1,fp8,fp8,0,0.21394240856170654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,8,128,1,float16,fp8,0,0.08516640067100525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,8,128,1,fp8,fp8,0,0.08440960049629212
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,48,128,1,float16,float16,0,0.06982240080833435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,48,128,1,float16,fp8,0,0.060262399911880496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,1,128,1,float16,float16,0,0.04335840046405792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,1,128,1,float16,fp8,0,0.04103519916534424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,1,128,1,fp8,fp8,0,0.040822398662567136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,2,128,1,float16,float16,0,0.04392800033092499
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,2,128,1,float16,fp8,0,0.041356799006462094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,2,128,1,fp8,fp8,0,0.04136480093002319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,4,128,1,float16,float16,0,0.04464319944381714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,4,128,1,float16,fp8,0,0.04268639981746673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,4,128,1,fp8,fp8,0,0.04248160123825073
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,48,8,128,1,float16,fp8,0,0.5981567859649658
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,8,128,1,float16,float16,0,0.04695520102977753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,8,128,1,float16,fp8,0,0.04535680115222931
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,48,128,1,float16,float16,0,0.04021919965744018
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,48,4,128,1,float16,fp8,0,0.15170880556106567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,48,128,1,fp8,fp8,0,0.03160319924354553
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,1,128,1,float16,float16,0,0.022593599557876588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,48,2,128,1,fp8,fp8,0,0.28625919818878176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,48,128,1,float16,fp8,0,0.11212320327758789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,2,128,1,float16,float16,0,0.02274399995803833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,2,128,1,float16,fp8,0,0.024035200476646423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,2,128,1,fp8,fp8,0,0.024105599522590636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,4,128,1,float16,float16,0,0.022971199452877046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,48,4,128,1,float16,float16,0,0.07963839769363404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,4,128,1,fp8,fp8,0,0.024132800102233887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,8,128,1,float16,float16,0,0.02380480021238327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,8,128,1,float16,fp8,0,0.024302400648593903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,8,128,1,fp8,fp8,0,0.02423679977655411
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,48,128,1,float16,float16,0,0.023051199316978455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,48,128,1,fp8,fp8,0,0.060369598865509036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,48,128,1,float16,fp8,0,0.01987680047750473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,48,128,1,fp8,fp8,0,0.02022880017757416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,1,128,1,float16,float16,0,0.017411200702190398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,1,128,1,float16,fp8,0,0.018564799427986146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,1,128,1,fp8,fp8,0,0.018580800294876097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,2,128,1,float16,float16,0,0.017654399573802947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,2,128,1,float16,fp8,0,0.01854719966650009
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,2,128,1,fp8,fp8,0,0.01852640062570572
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,4,128,1,float16,float16,0,0.017638400197029114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,4,128,1,float16,fp8,0,0.01857600063085556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,4,128,1,fp8,fp8,0,0.018515199422836304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,8,128,1,float16,float16,0,0.018110400438308714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,8,128,1,float16,fp8,0,0.01849759966135025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,48,8,128,1,fp8,fp8,0,0.018569600582122803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,48,128,1,float16,float16,0,0.014867199957370758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,48,128,1,float16,fp8,0,0.014374400675296783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,1,128,1,float16,float16,0,0.01281919926404953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,48,8,128,1,fp8,fp8,0,0.04477440118789673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,1,128,1,float16,fp8,0,0.013678400218486786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,1,128,1,fp8,fp8,0,0.013385599851608277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,2,128,1,float16,float16,0,0.01284479945898056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,48,128,1,float16,fp8,0,0.030904000997543334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,4,128,1,float16,float16,0,0.013145600259304047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,4,128,1,float16,fp8,0,0.01329600065946579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,4,128,1,fp8,fp8,0,0.013419200479984284
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,8,128,1,float16,float16,0,0.013035200536251068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,8,128,1,float16,fp8,0,0.013649600744247436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,8,128,1,fp8,fp8,0,0.013380800187587739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,48,128,1,float16,float16,0,0.014019200205802917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,48,128,1,float16,fp8,0,0.013777600228786468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,48,128,1,fp8,fp8,0,0.013875199854373932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,1,128,1,float16,float16,0,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,1,128,1,fp8,fp8,0,0.013193599879741669
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,2,128,1,float16,float16,0,0.012556800246238708
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,2,128,1,float16,fp8,0,0.013036799430847169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,2,128,1,fp8,fp8,0,0.012992000579833985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,4,128,1,float16,fp8,0,0.024052800238132478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,4,128,1,float16,fp8,0,0.013145600259304047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,4,128,1,fp8,fp8,0,0.0131632000207901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,8,128,1,float16,float16,0,0.01281919926404953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,8,128,1,float16,fp8,0,0.013201600313186646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,8,128,1,fp8,fp8,0,0.013185599446296692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,48,1,128,1,float16,float16,0,0.2659584045410156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,48,1,128,1,float16,fp8,0,0.28590400218963624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,48,128,1,fp8,fp8,0,0.014127999544143677
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,48,1,128,1,fp8,fp8,0,0.28608319759368894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,2,128,1,float16,fp8,0,0.013471999764442444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,48,2,128,1,fp8,fp8,0,0.013494400680065155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,48,2,128,1,float16,fp8,0,0.2870480060577393
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,1,128,1,float16,fp8,0,0.02404800057411194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,48,2,128,1,fp8,fp8,0,0.2840464115142822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,48,4,128,1,float16,float16,0,0.2765647888183594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,48,4,128,1,float16,fp8,0,0.2937999963760376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,1,128,1,float16,fp8,0,0.013483199477195739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,48,4,128,1,fp8,fp8,0,0.2931663990020752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,48,8,128,1,float16,float16,0,0.28652000427246094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,48,8,128,1,float16,fp8,0,0.3066751956939697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,48,8,128,1,fp8,fp8,0,0.3064255952835083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,48,4,128,1,float16,float16,0,0.012558400630950928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,1,128,1,float16,float16,0,0.13916319608688354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,48,128,1,float16,fp8,0,0.31469600200653075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,1,128,1,float16,fp8,0,0.15059679746627808
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,2,128,1,float16,float16,0,0.13977919816970824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,2,128,1,float16,fp8,0,0.1527616024017334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,2,128,1,fp8,fp8,0,0.14945759773254394
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,4,128,1,float16,float16,0,0.14417439699172974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,4,128,1,float16,fp8,0,0.1546336054801941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,4,128,1,fp8,fp8,0,0.15288000106811522
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,8,128,1,float16,fp8,0,0.1609536051750183
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,8,128,1,fp8,fp8,0,0.15964159965515137
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,48,128,1,float16,float16,0,0.1704367995262146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,48,128,1,float16,fp8,0,0.16507999897003173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,48,128,1,fp8,fp8,0,0.16519520282745362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,1,128,1,float16,float16,0,0.07733920216560364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,1,128,1,float16,fp8,0,0.08109599947929383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,1,128,1,fp8,fp8,0,0.08125280141830445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,2,128,1,float16,float16,0,0.07818080186843872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,48,2,128,1,float16,float16,0,0.26571040153503417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,2,128,1,float16,fp8,0,0.08195199966430664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,2,128,1,fp8,fp8,0,0.08087360262870788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,4,128,1,float16,fp8,0,0.08293920159339904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,48,1,128,1,fp8,fp8,0,0.023982399702072145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,4,128,1,fp8,fp8,0,0.08260480165481568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,8,128,1,float16,float16,0,0.08347200155258179
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,8,128,1,float16,fp8,0,0.08523359894752502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,8,128,1,fp8,fp8,0,0.08574720025062561
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,48,128,1,float16,float16,0,0.09363359808921815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,48,128,1,float16,fp8,0,0.08718240261077881
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,48,128,1,fp8,fp8,0,0.0870191991329193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,48,128,1,float16,float16,0,0.32371039390563966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,1,128,1,float16,float16,0,0.04382559955120087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,48,128,1,fp8,fp8,0,0.31353919506072997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,2,128,1,float16,float16,0,0.04422720074653626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,1,128,1,fp8,fp8,0,0.14943679571151733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,2,128,1,fp8,fp8,0,0.04164479970932007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,4,128,1,float16,float16,0,0.04526880085468292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,4,128,1,float16,fp8,0,0.043115198612213135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,4,128,1,fp8,fp8,0,0.04294399917125702
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,8,128,1,float16,fp8,0,0.04582560062408447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,48,8,128,1,float16,float16,0,0.1509071946144104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,8,128,1,fp8,fp8,0,0.0455487996339798
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,48,128,1,float16,float16,0,0.0517632007598877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,48,128,1,float16,fp8,0,0.045388799905776975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,48,128,1,fp8,fp8,0,0.04499039947986603
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,1,128,1,float16,fp8,0,0.024060800671577454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,1,128,1,fp8,fp8,0,0.0241007998585701
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,2,128,1,float16,float16,0,0.022889600694179536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,2,128,1,float16,fp8,0,0.024238400161266327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,2,128,1,fp8,fp8,0,0.02414720058441162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,4,128,1,float16,float16,0,0.023051199316978455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,4,128,1,float16,fp8,0,0.024209600687026978
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,4,128,1,fp8,fp8,0,0.024208000302314757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,48,4,128,1,float16,float16,0,0.08024320006370544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,8,128,1,float16,fp8,0,0.024377599358558655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,8,128,1,fp8,fp8,0,0.02434239983558655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,48,128,1,float16,float16,0,0.026966398954391478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,48,128,1,float16,fp8,0,0.024374400079250336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,48,128,1,fp8,fp8,0,0.02438720017671585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,1,128,1,float16,float16,0,0.017392000555992125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,1,128,1,float16,fp8,0,0.018667200207710268
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,1,128,1,float16,fp8,0,0.041366401314735415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,1,128,1,fp8,fp8,0,0.01857600063085556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,2,128,1,float16,float16,0,0.01720000058412552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,2,128,1,float16,fp8,0,0.0186831995844841
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,2,128,1,float16,fp8,0,0.04175840020179748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,4,128,1,float16,float16,0,0.017552000284194947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,4,128,1,float16,fp8,0,0.018641600012779237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,8,128,1,float16,float16,0,0.01783519983291626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,8,128,1,float16,float16,0,0.04763680100440979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,8,128,1,float16,fp8,0,0.01857919991016388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,8,128,1,fp8,fp8,0,0.018691200017929076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,48,128,1,float16,float16,0,0.01804479956626892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,48,128,1,float16,fp8,0,0.018559999763965607
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,1,128,1,float16,float16,0,0.012718400359153748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,1,128,1,float16,fp8,0,0.013569599390029908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,1,128,1,fp8,fp8,0,0.013246400654315949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,2,128,1,float16,float16,0,0.012718400359153748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,1,128,1,float16,float16,0,0.022569599747657775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,2,128,1,float16,fp8,0,0.01342879980802536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,2,128,1,fp8,fp8,0,0.01329440027475357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,4,128,1,float16,float16,0,0.012809599936008453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,4,128,1,fp8,fp8,0,0.013097600638866424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,8,128,1,float16,fp8,0,0.013500800728797913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,8,128,1,fp8,fp8,0,0.013380800187587739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,48,128,1,float16,float16,0,0.013337600231170654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,48,128,1,float16,fp8,0,0.013510400056838989
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,48,128,1,fp8,fp8,0,0.013655999302864074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,1,128,1,float16,float16,0,0.012521600723266602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,1,128,1,float16,fp8,0,0.013025599718093871
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,1,128,1,fp8,fp8,0,0.013016000390052795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,48,8,128,1,float16,float16,0,0.023721599578857423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,2,128,1,float16,float16,0,0.012387199699878693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,2,128,1,float16,fp8,0,0.013308799266815186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,4,128,1,float16,float16,0,0.012654399871826172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,4,128,1,float16,fp8,0,0.013220800459384919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,4,128,1,fp8,fp8,0,0.013385599851608277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,8,128,1,float16,float16,0,0.012734399735927581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,8,128,1,float16,fp8,0,0.013100799918174744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,8,128,1,fp8,fp8,0,0.013097600638866424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,48,128,1,float16,float16,0,0.013326400518417358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,48,128,1,float16,fp8,0,0.013537600636482239
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,48,128,1,fp8,fp8,0,0.013478399813175201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,1,128,1,float16,float16,0,0.012356799840927125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,4,128,1,fp8,fp8,0,0.018537600338459016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,1,128,1,float16,fp8,0,0.013209599256515502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,2,128,1,float16,float16,0,0.012510399520397186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,2,128,1,float16,fp8,0,0.012985600531101227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,2,128,1,fp8,fp8,0,0.012993599474430084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,4,128,1,float16,float16,0,0.012187200039625168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,4,128,1,float16,fp8,0,0.013007999956607818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,4,128,1,fp8,fp8,0,0.013156799972057343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,8,128,1,float16,float16,0,0.012481600046157837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,8,128,1,float16,fp8,0,0.013046400249004364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,8,128,1,fp8,fp8,0,0.01313920021057129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,8,128,1,float16,float16,0,0.012942400574684144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,48,1,128,1,float16,float16,0,0.13714879751205444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,48,1,128,1,float16,fp8,0,0.1518623948097229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,48,1,128,1,fp8,fp8,0,0.15210399627685547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,48,2,128,1,float16,float16,0,0.14069600105285646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,48,2,128,1,float16,fp8,0,0.15147520303726197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,48,2,128,1,fp8,fp8,0,0.15056159496307372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,48,4,128,1,float16,float16,0,0.14504640102386473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,48,1,128,1,fp8,fp8,0,0.04086079895496368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,48,2,128,1,fp8,fp8,0,0.018475200235843658
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,48,4,128,1,fp8,fp8,0,0.15352959632873536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,48,8,128,1,float16,float16,0,0.17982399463653564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,48,8,128,1,float16,fp8,0,0.18958719968795776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,48,8,128,1,fp8,fp8,0,0.1904960036277771
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,48,128,1,fp8,fp8,0,0.018510399758815764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,48,128,1,float16,float16,0,0.2596127986907959
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,48,128,1,float16,fp8,0,0.270796799659729
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,48,128,1,fp8,fp8,0,0.2708911895751953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,48,4,128,1,float16,fp8,0,0.013116799294948578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,1,128,1,float16,float16,0,0.07729920148849487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,1,128,1,float16,fp8,0,0.08155199885368347
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,1,128,1,fp8,fp8,0,0.08187199831008911
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,2,128,1,float16,float16,0,0.07781599760055542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,2,128,1,fp8,fp8,0,0.08236960172653199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,4,128,1,float16,float16,0,0.07956640124320984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,4,128,1,float16,fp8,0,0.08341119885444641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,4,128,1,fp8,fp8,0,0.08342239856719971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,8,128,1,float16,float16,0,0.09761760234832764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,8,128,1,float16,fp8,0,0.1020799994468689
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,8,128,1,fp8,fp8,0,0.10170719623565674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,48,128,1,float16,float16,0,0.13825759887695313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,48,128,1,float16,fp8,0,0.14296480417251586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,48,128,1,fp8,fp8,0,0.14203840494155884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,1,128,1,float16,float16,0,0.04366880059242249
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,1,128,1,fp8,fp8,0,0.04147840142250061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,2,128,1,float16,float16,0,0.044180798530578616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,48,2,128,1,fp8,fp8,0,0.013171200454235078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,2,128,1,fp8,fp8,0,0.04223200082778931
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,4,128,1,float16,float16,0,0.04601919949054718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,4,128,1,float16,fp8,0,0.04318079948425293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,4,128,1,fp8,fp8,0,0.04296480119228363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,8,128,1,float16,float16,0,0.055302399396896365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,8,128,1,float16,fp8,0,0.05432959794998169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,8,128,1,fp8,fp8,0,0.05385919809341431
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,48,128,1,float16,float16,0,0.0741216003894806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,48,128,1,float16,fp8,0,0.07355039715766906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,48,128,1,fp8,fp8,0,0.07384480237960815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,1,128,1,float16,fp8,0,0.024270400404930115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,1,128,1,fp8,fp8,0,0.024243199825286867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,2,128,1,float16,float16,0,0.022977599501609804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,2,128,1,float16,fp8,0,0.024344000220298766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,48,1,128,1,fp8,fp8,0,0.012982399761676788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,2,128,1,fp8,fp8,0,0.024439999461174013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,4,128,1,float16,float16,0,0.022889600694179536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,4,128,1,float16,fp8,0,0.024542400240898134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,4,128,1,fp8,fp8,0,0.024396799504756927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,8,128,1,float16,float16,0,0.02712959945201874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,8,128,1,float16,fp8,0,0.02908799946308136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,8,128,1,fp8,fp8,0,0.029311999678611755
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,48,128,1,float16,float16,0,0.03758879899978638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,48,128,1,float16,fp8,0,0.038176000118255615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,48,128,1,fp8,fp8,0,0.038011199235916136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,1,128,1,float16,float16,0,0.017283199727535246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,1,128,1,float16,fp8,0,0.018510399758815764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,1,128,1,fp8,fp8,0,0.01842560023069382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,2,128,1,float16,float16,0,0.017470400035381316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,2,128,1,float16,fp8,0,0.018486399948596955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,2,128,1,fp8,fp8,0,0.018587200343608855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,4,128,1,float16,float16,0,0.0174687996506691
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,4,128,1,float16,fp8,0,0.018580800294876097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,4,128,1,fp8,fp8,0,0.018654400110244752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,8,128,1,float16,float16,0,0.0173567995429039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,8,128,1,float16,fp8,0,0.01876640021800995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,48,8,128,1,fp8,fp8,0,0.018500800430774688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,48,128,1,float16,float16,0,0.022086399793624877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,48,128,1,float16,fp8,0,0.023235200345516203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,48,128,1,fp8,fp8,0,0.023148800432682037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,1,128,1,float16,float16,0,0.01289760023355484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,1,128,1,float16,fp8,0,0.013388800621032714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,1,128,1,float16,fp8,0,0.04174239933490753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,2,128,1,float16,float16,0,0.01284160017967224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,2,128,1,float16,fp8,0,0.013603200018405915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,48,2,128,1,float16,fp8,0,0.04184960126876831
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,4,128,1,float16,float16,0,0.013017599284648896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,4,128,1,fp8,fp8,0,0.013708800077438354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,8,128,1,float16,float16,0,0.013020800054073333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,8,128,1,float16,fp8,0,0.013627199828624726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,8,128,1,fp8,fp8,0,0.013622400164604188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,48,128,1,float16,float16,0,0.017396800220012665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,48,128,1,float16,fp8,0,0.01801439970731735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,48,128,1,fp8,fp8,0,0.0179967999458313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,1,128,1,float16,float16,0,0.01231520026922226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,1,128,1,float16,fp8,0,0.013171200454235078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,48,1,128,1,float16,float16,0,0.022380800545215608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,2,128,1,float16,fp8,0,0.013337600231170654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,48,4,128,1,float16,fp8,0,0.15341440439224244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,4,128,1,float16,float16,0,0.012608000636100769
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,4,128,1,float16,fp8,0,0.013225600123405457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,4,128,1,fp8,fp8,0,0.013240000605583191
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,8,128,1,float16,float16,0,0.01239359974861145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,8,128,1,float16,fp8,0,0.013052800297737121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,48,2,128,1,float16,fp8,0,0.08225759863853455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,8,128,1,fp8,fp8,0,0.01340479999780655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,48,128,1,float16,float16,0,0.013278399407863618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,48,128,1,float16,fp8,0,0.013655999302864074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,48,128,1,fp8,fp8,0,0.013443200290203095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,1,128,1,float16,float16,0,0.01234079971909523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,1,128,1,fp8,fp8,0,0.013742400705814362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,1,128,1,float16,fp8,0,0.013075199723243714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,1,128,1,fp8,fp8,0,0.012998400628566742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,4,128,1,float16,fp8,0,0.013433599472045898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,2,128,1,fp8,fp8,0,0.013079999387264252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,4,128,1,float16,float16,0,0.012350399792194367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,4,128,1,float16,fp8,0,0.013062399625778199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,4,128,1,fp8,fp8,0,0.013025599718093871
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,8,128,1,float16,float16,0,0.012203200161457062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,1,128,1,fp8,fp8,0,0.012919999659061432
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,8,128,1,float16,fp8,0,0.013055999577045441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,2,128,1,float16,float16,0,0.012387199699878693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,48,128,1,float16,float16,0,0.013092799484729767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,48,128,1,fp8,fp8,0,0.013348799943923951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,48,2,128,1,fp8,fp8,0,0.013264000415802002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,1,128,1,float16,fp8,0,0.012774400413036346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,1,128,1,fp8,fp8,0,0.012753599882125854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,2,128,1,float16,float16,0,0.012100800126791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,2,128,1,float16,fp8,0,0.012807999551296235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,2,128,1,fp8,fp8,0,0.012800000607967377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,4,128,1,float16,float16,0,0.012132800370454788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,4,128,1,float16,fp8,0,0.012856000661849975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,4,128,1,fp8,fp8,0,0.01286720037460327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,8,128,1,float16,float16,0,0.012139199674129486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,8,128,1,float16,fp8,0,0.012788799405097962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,8,128,1,fp8,fp8,0,0.012675200402736665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,48,1,128,1,float16,float16,0,0.07723199725151061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,48,1,128,1,float16,fp8,0,0.08252000212669372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,48,1,128,1,fp8,fp8,0,0.0826416015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,48,2,128,1,float16,float16,0,0.07951359748840332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,48,2,128,1,float16,fp8,0,0.08315680027008057
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,48,2,128,1,fp8,fp8,0,0.08331999778747559
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,48,4,128,1,float16,float16,0,0.09409279823303222
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,2,128,1,float16,float16,0,0.012379200011491776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,48,4,128,1,float16,fp8,0,0.10150079727172852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,2,128,1,float16,fp8,0,0.013019199669361114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,48,4,128,1,fp8,fp8,0,0.10049439668655395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,48,8,128,1,float16,float16,0,0.09943519830703736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,48,8,128,1,float16,fp8,0,0.10405759811401367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,48,8,128,1,fp8,fp8,0,0.10420639514923095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,48,128,1,float16,float16,0,0.22895519733428954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,48,128,1,float16,fp8,0,0.2527551889419556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,48,128,1,fp8,fp8,0,0.2542704105377197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,1,128,1,float16,fp8,0,0.04196160137653351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,1,128,1,fp8,fp8,0,0.04174720048904419
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,2,128,1,float16,float16,0,0.045307201147079465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,48,8,128,1,fp8,fp8,0,0.013012799620628356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,2,128,1,fp8,fp8,0,0.0425024002790451
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,4,128,1,float16,float16,0,0.05332800149917603
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,48,128,1,float16,fp8,0,0.01321599930524826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,4,128,1,fp8,fp8,0,0.052478402853012085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,8,128,1,float16,float16,0,0.05592640042304993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,8,128,1,float16,fp8,0,0.05534240007400513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,48,128,1,float16,float16,0,0.12120959758758545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,48,128,1,float16,fp8,0,0.12912479639053345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,48,128,1,fp8,fp8,0,0.13113119602203369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,1,128,1,float16,float16,0,0.02282399982213974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,1,128,1,float16,fp8,0,0.0249439999461174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,2,128,1,float16,float16,0,0.023423999547958374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,2,128,1,float16,fp8,0,0.025040000677108765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,2,128,1,fp8,fp8,0,0.025043201446533204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,4,128,1,float16,float16,0,0.026662400364875792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,4,128,1,float16,fp8,0,0.029420799016952513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,4,128,1,fp8,fp8,0,0.029297599196434022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,8,128,1,float16,float16,0,0.0272816002368927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,8,128,1,float16,fp8,0,0.02937920093536377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,48,2,128,1,fp8,fp8,0,0.013433599472045898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,48,128,1,float16,float16,0,0.05911840200424194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,48,128,1,float16,fp8,0,0.06589599847793579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,1,128,1,float16,float16,0,0.04545600116252899
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,1,128,1,float16,float16,0,0.01780640035867691
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,1,128,1,float16,fp8,0,0.01897920072078705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,2,128,1,float16,fp8,0,0.042423999309539794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,1,128,1,fp8,fp8,0,0.01902720034122467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,2,128,1,float16,float16,0,0.017795200645923614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,2,128,1,float16,fp8,0,0.0190528005361557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,4,128,1,float16,fp8,0,0.052236801385879515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,48,1,128,1,float16,float16,0,0.012179200351238251
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,4,128,1,float16,fp8,0,0.018934400379657747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,4,128,1,fp8,fp8,0,0.01889120042324066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,8,128,1,float16,float16,0,0.018156799674034118
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,8,128,1,float16,fp8,0,0.01878879964351654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,48,128,1,float16,float16,0,0.0331712007522583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,48,128,1,float16,fp8,0,0.036976000666618346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,48,128,1,fp8,fp8,0,0.0368943989276886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,1,128,1,float16,float16,0,0.012748800218105316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,1,128,1,float16,fp8,0,0.013420799374580383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,1,128,1,fp8,fp8,0,0.013302400708198547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,2,128,1,float16,float16,0,0.012800000607967377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,2,128,1,float16,fp8,0,0.0133775994181633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,2,128,1,fp8,fp8,0,0.013620799779891968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,4,128,1,float16,float16,0,0.012836800515651703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,4,128,1,float16,fp8,0,0.01340479999780655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,4,128,1,fp8,fp8,0,0.013353599607944489
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,8,128,1,fp8,fp8,0,0.029516801238059998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,8,128,1,float16,float16,0,0.013225600123405457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,8,128,1,float16,fp8,0,0.013470399379730224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,48,8,128,1,fp8,fp8,0,0.013337600231170654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,48,128,1,fp8,fp8,0,0.06585919857025146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,48,128,1,float16,float16,0,0.02112639993429184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,48,128,1,float16,fp8,0,0.022835199534893037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,48,128,1,fp8,fp8,0,0.022628800570964815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,1,128,1,float16,float16,0,0.012508800625801087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,1,128,1,float16,fp8,0,0.01327199935913086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,1,128,1,fp8,fp8,0,0.013284799456596375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,2,128,1,float16,float16,0,0.012479999661445617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,2,128,1,fp8,fp8,0,0.018828800320625304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,2,128,1,fp8,fp8,0,0.013312000036239623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,4,128,1,float16,float16,0,0.017601600289344786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,4,128,1,float16,float16,0,0.01231520026922226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,48,8,128,1,fp8,fp8,0,0.0553167998790741
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,8,128,1,float16,float16,0,0.01226240023970604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,8,128,1,float16,fp8,0,0.01321759968996048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,8,128,1,fp8,fp8,0,0.013150399923324585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,48,128,1,float16,float16,0,0.01687840074300766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,48,128,1,float16,fp8,0,0.017790399491786957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,48,128,1,fp8,fp8,0,0.0176816001534462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,1,128,1,float16,fp8,0,0.013191999495029449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,1,128,1,fp8,fp8,0,0.01318880021572113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,2,128,1,float16,float16,0,0.0124719999730587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,48,1,128,1,fp8,fp8,0,0.024648000299930573
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,2,128,1,fp8,fp8,0,0.012985600531101227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,4,128,1,float16,float16,0,0.012081599980592727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,4,128,1,float16,fp8,0,0.012886400520801543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,4,128,1,fp8,fp8,0,0.013155199587345123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,8,128,1,float16,float16,0,0.012108799815177918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,8,128,1,float16,fp8,0,0.012969599664211273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,8,128,1,fp8,fp8,0,0.012902399897575379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,48,128,1,float16,float16,0,0.013291199505329133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,48,128,1,float16,fp8,0,0.0134320005774498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,48,128,1,fp8,fp8,0,0.013276800513267517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,1,128,1,float16,float16,0,0.011931200325489045
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,1,128,1,float16,fp8,0,0.012721599638462066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,1,128,1,fp8,fp8,0,0.01271360069513321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,2,128,1,float16,float16,0,0.01194240003824234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,2,128,1,float16,fp8,0,0.012759999930858612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,2,128,1,fp8,fp8,0,0.0127920001745224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,4,128,1,float16,float16,0,0.012150400131940842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,4,128,1,float16,fp8,0,0.012838399410247803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,4,128,1,fp8,fp8,0,0.012863999605178833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,8,128,1,float16,float16,0,0.012015999853610992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,8,128,1,float16,fp8,0,0.0125231996178627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,48,8,128,1,fp8,fp8,0,0.012644800543785095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,4,128,1,float16,fp8,0,0.013176000118255616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,48,128,1,float16,float16,0,0.013068799674510957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,48,128,1,float16,fp8,0,0.013283200562000275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,48,128,1,fp8,fp8,0,0.013335999846458436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,1,128,1,float16,float16,0,0.011900799721479416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,1,128,1,float16,fp8,0,0.01276479959487915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,1,128,1,fp8,fp8,0,0.012534399330615998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,2,128,1,float16,float16,0,0.01175680011510849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,1,128,1,float16,float16,0,0.012334399670362473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,2,128,1,fp8,fp8,0,0.012491200119256973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,4,128,1,float16,float16,0,0.011988800019025803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,4,128,1,fp8,fp8,0,0.012694400548934937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,8,128,1,float16,float16,0,0.011811199784278869
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,8,128,1,float16,fp8,0,0.012479999661445617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,8,128,1,fp8,fp8,0,0.01249760016798973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,48,1,128,1,float16,float16,0,0.021348799765110015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,48,1,128,1,float16,fp8,0,0.022676800191402436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,48,1,128,1,fp8,fp8,0,0.023228800296783446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,48,2,128,1,float16,float16,0,0.028174400329589844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,48,2,128,1,float16,fp8,0,0.03180800080299377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,48,2,128,1,fp8,fp8,0,0.031350401043891904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,48,4,128,1,float16,float16,0,0.04335359930992126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,48,4,128,1,float16,fp8,0,0.04911839962005615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,48,4,128,1,fp8,fp8,0,0.04939039945602417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,48,8,128,1,float16,float16,0,0.07128000259399414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,48,8,128,1,float16,fp8,0,0.08492479920387268
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,48,8,128,1,fp8,fp8,0,0.0845359981060028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,48,128,1,float16,float16,0,0.1836575984954834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,48,128,1,float16,fp8,0,0.22546238899230958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,1,128,1,float16,float16,0,0.014763200283050537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,48,128,1,fp8,fp8,0,0.22525599002838134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,1,128,1,float16,fp8,0,0.015516799688339234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,1,128,1,fp8,fp8,0,0.015908800065517426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,2,128,1,float16,float16,0,0.018287999927997588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,2,128,1,float16,fp8,0,0.020231999456882477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,2,128,1,fp8,fp8,0,0.01993599981069565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,4,128,1,float16,float16,0,0.026123198866844177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,4,128,1,float16,fp8,0,0.029054400324821473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,4,128,1,fp8,fp8,0,0.029183998703956604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,8,128,1,float16,float16,0,0.04040960073471069
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,8,128,1,float16,fp8,0,0.046856001019477844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,48,8,128,1,fp8,fp8,0,0.046758401393890384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,48,128,1,float16,float16,0,0.09934719800949096
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,48,128,1,float16,fp8,0,0.11899199485778808
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,48,128,1,fp8,fp8,0,0.11913119554519654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,1,128,1,float16,fp8,0,0.014070400595664978
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,2,128,1,float16,fp8,0,0.012511999905109405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,2,128,1,float16,float16,0,0.013569599390029908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,2,128,1,float16,fp8,0,0.014375999569892883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,2,128,1,fp8,fp8,0,0.014302399754524232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,48,2,128,1,float16,fp8,0,0.013187199831008911
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,4,128,1,float16,float16,0,0.017454400658607483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,4,128,1,float16,fp8,0,0.01897599995136261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,4,128,1,fp8,fp8,0,0.018883199989795686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,8,128,1,float16,float16,0,0.024899199604988098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,8,128,1,float16,fp8,0,0.027723199129104613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,8,128,1,fp8,fp8,0,0.02755039930343628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,48,128,1,float16,float16,0,0.0548367977142334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,48,128,1,float16,fp8,0,0.06395679712295532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,48,128,1,fp8,fp8,0,0.06398079991340637
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,1,128,1,float16,float16,0,0.012585599720478059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,1,128,1,float16,fp8,0,0.013196800649166108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,1,128,1,fp8,fp8,0,0.013264000415802002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,2,128,1,float16,float16,0,0.012428800016641617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,2,128,1,float16,fp8,0,0.013308799266815186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,2,128,1,fp8,fp8,0,0.013255999982357025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,4,128,1,float16,float16,0,0.012582400441169738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,4,128,1,float16,fp8,0,0.013569599390029908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,4,128,1,fp8,fp8,0,0.013555200397968292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,8,128,1,float16,float16,0,0.01645279973745346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,8,128,1,float16,fp8,0,0.017895999550819396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,48,8,128,1,fp8,fp8,0,0.01790879964828491
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,4,128,1,fp8,fp8,0,0.01324319988489151
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,48,8,128,1,fp8,fp8,0,0.018991999328136444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,1,128,1,float16,float16,0,0.013184000551700593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,48,128,1,fp8,fp8,0,0.03623520135879517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,48,1,128,1,fp8,fp8,0,0.014083200693130493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,1,128,1,float16,fp8,0,0.012675200402736665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,1,128,1,fp8,fp8,0,0.012703999876976013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,2,128,1,float16,fp8,0,0.012723200023174286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,2,128,1,fp8,fp8,0,0.012796799838542938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,4,128,1,float16,float16,0,0.012081599980592727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,4,128,1,float16,fp8,0,0.012868799269199371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,4,128,1,fp8,fp8,0,0.012740799784660339
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,8,128,1,float16,float16,0,0.012078399956226348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,8,128,1,float16,fp8,0,0.012939199805259705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,8,128,1,fp8,fp8,0,0.012945599853992462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,48,128,1,float16,float16,0,0.02074880003929138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,48,128,1,float16,fp8,0,0.022337600588798523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,48,128,1,fp8,fp8,0,0.022356800734996796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,1,128,1,float16,float16,0,0.01183359995484352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,1,128,1,float16,fp8,0,0.012630400061607362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,1,128,1,fp8,fp8,0,0.012619200348854064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,2,128,1,float16,float16,0,0.011990399658679962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,2,128,1,float16,fp8,0,0.012636800110340119
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,2,128,1,fp8,fp8,0,0.01266079992055893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,4,128,1,float16,float16,0,0.01191840022802353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,4,128,1,float16,fp8,0,0.01265919953584671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,4,128,1,fp8,fp8,0,0.012681600451469422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,8,128,1,float16,float16,0,0.01202239990234375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,8,128,1,float16,fp8,0,0.01284320056438446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,48,8,128,1,fp8,fp8,0,0.012878400087356568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,48,128,1,float16,float16,0,0.016659200191497803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,48,128,1,float16,fp8,0,0.01738079935312271
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,48,128,1,fp8,fp8,0,0.01748320013284683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,1,128,1,float16,float16,0,0.01159520000219345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,48,128,1,float16,fp8,0,0.03608640134334564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,1,128,1,float16,fp8,0,0.012275200337171555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,1,128,1,fp8,fp8,0,0.012161599844694138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,2,128,1,float16,fp8,0,0.012206400185823441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,2,128,1,fp8,fp8,0,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,48,4,128,1,float16,fp8,0,0.012511999905109405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,4,128,1,float16,float16,0,0.011499200016260147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,4,128,1,float16,fp8,0,0.012217599898576736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,4,128,1,fp8,fp8,0,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,8,128,1,float16,float16,0,0.011699199676513672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,8,128,1,float16,fp8,0,0.012651200592517852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,8,128,1,fp8,fp8,0,0.012707200646400452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,48,128,1,float16,float16,0,0.013025599718093871
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,48,128,1,float16,fp8,0,0.013252800703048706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,48,128,1,fp8,fp8,0,0.013249599933624267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,1,128,1,float16,float16,0,0.011392000317573547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,1,128,1,float16,fp8,0,0.012216000258922577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,1,128,1,fp8,fp8,0,0.012176000326871873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,2,128,1,float16,float16,0,0.011528000235557556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,2,128,1,float16,fp8,0,0.01223680004477501
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,2,128,1,fp8,fp8,0,0.012318400293588638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,4,128,1,float16,float16,0,0.011428800225257874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,4,128,1,float16,fp8,0,0.012435200065374375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,4,128,1,fp8,fp8,0,0.012064000219106674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,8,128,1,float16,float16,0,0.011751999706029892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,8,128,1,float16,fp8,0,0.012243200093507767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,48,8,128,1,fp8,fp8,0,0.012464000284671784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,48,128,1,float16,float16,0,0.012700800597667695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,48,128,1,float16,fp8,0,0.013204799592494964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,48,128,1,fp8,fp8,0,0.012996800243854523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,1,128,1,float16,float16,0,0.011529599875211715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,1,128,1,float16,fp8,0,0.012043199688196182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,1,128,1,fp8,fp8,0,0.012225600332021714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,2,128,1,float16,float16,0,0.011425600200891495
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,2,128,1,float16,fp8,0,0.012225600332021714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,2,128,1,fp8,fp8,0,0.012097600102424621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,4,128,1,float16,float16,0,0.011454399675130844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,4,128,1,float16,fp8,0,0.012692800164222718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,4,128,1,fp8,fp8,0,0.01218400001525879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,8,128,1,float16,float16,0,0.011358399689197541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,8,128,1,float16,fp8,0,0.012177599966526032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,48,8,128,1,fp8,fp8,0,0.012070400267839431
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,48,2,128,1,float16,fp8,0,0.01316159963607788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,48,128,1,float16,float16,0,0.03242399990558624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,48,2,128,1,float16,float16,0,0.011584000289440155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,2,128,1,float16,float16,0,0.012017600238323212
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,48,1,128,1,float16,float16,0,0.011803200095891952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,40,1,128,1,float16,fp8,0,32.305780029296876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,40,1,128,1,fp8,fp8,0,32.1946044921875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,40,2,128,1,float16,fp8,0,32.0456298828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,40,2,128,1,fp8,fp8,0,32.256143188476564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,40,1,128,1,float16,float16,0,38.69833984375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,40,2,128,1,float16,float16,0,39.065771484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,40,4,128,1,float16,fp8,0,31.823760986328125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,40,4,128,1,float16,float16,0,39.85755920410156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,1,128,1,float16,float16,0,19.43309326171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,40,4,128,1,fp8,fp8,0,32.688967895507815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,40,8,128,1,float16,fp8,0,32.53859252929688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,40,8,128,1,fp8,fp8,0,32.626373291015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,40,8,128,1,float16,float16,0,39.53663635253906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,1,128,1,float16,fp8,0,16.475299072265624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,1,128,1,fp8,fp8,0,16.80962677001953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,2,128,1,float16,fp8,0,16.51014404296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,2,128,1,float16,float16,0,19.91447296142578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,2,128,1,fp8,fp8,0,16.460667419433594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,4,128,1,float16,fp8,0,16.315838623046876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,4,128,1,fp8,fp8,0,16.273342895507813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,4,128,1,float16,float16,0,19.891744995117186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,8,128,1,float16,float16,0,19.429443359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,8,128,1,float16,fp8,0,16.553150939941407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,8,128,1,fp8,fp8,0,16.89250946044922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,1,128,1,float16,float16,0,10.306161499023437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,1,128,1,float16,fp8,0,8.191854095458984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,1,128,1,fp8,fp8,0,8.121564483642578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,2,128,1,float16,fp8,0,8.124237060546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,2,128,1,float16,float16,0,9.857167816162109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,2,128,1,fp8,fp8,0,8.471609497070313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,4,128,1,float16,float16,0,9.901918029785156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,4,128,1,float16,fp8,0,8.376329803466797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,4,128,1,fp8,fp8,0,8.195098876953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,8,128,1,float16,float16,0,10.09322738647461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,8,128,1,float16,fp8,0,8.30580825805664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,8,128,1,fp8,fp8,0,8.190099334716797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,40,128,1,float16,fp8,0,4.082567977905273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,40,128,1,fp8,fp8,0,8.218478393554687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,40,128,1,fp8,fp8,0,4.12907829284668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,40,128,1,float16,fp8,0,8.225707244873046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,1,128,1,float16,float16,0,4.6814624786376955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,40,128,1,float16,float16,0,4.888420867919922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,1,128,1,float16,fp8,0,3.9586910247802733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,1,128,1,fp8,fp8,0,4.003577423095703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,40,40,128,1,float16,float16,0,9.71666259765625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,40,128,1,float16,fp8,0,16.462840270996093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,2,128,1,float16,float16,0,4.728155136108398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,2,128,1,float16,fp8,0,4.116188812255859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,4,128,1,float16,float16,0,4.8136943817138675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,40,128,1,fp8,fp8,0,16.36882629394531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,4,128,1,fp8,fp8,0,3.989934539794922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,40,40,128,1,float16,float16,0,19.658122253417968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,8,128,1,float16,float16,0,4.764945602416992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,2,128,1,fp8,fp8,0,4.0214286804199215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,8,128,1,float16,fp8,0,4.04891357421875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,4,128,1,float16,fp8,0,4.050161743164063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,40,8,128,1,fp8,fp8,0,4.14788818359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,40,1,128,1,float16,fp8,0,18.92815246582031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,40,1,128,1,fp8,fp8,0,18.70926513671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,40,2,128,1,float16,fp8,0,18.508442687988282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,40,2,128,1,fp8,fp8,0,18.546585083007812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,40,1,128,1,float16,float16,0,22.285296630859374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,40,4,128,1,float16,fp8,0,19.27205047607422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,40,2,128,1,float16,float16,0,22.76481475830078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,40,4,128,1,float16,float16,0,21.68224334716797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,40,128,1,float16,fp8,0,9.575202941894531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,40,128,1,float16,float16,0,11.403582763671874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,40,128,1,fp8,fp8,0,9.463756561279297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,1,128,1,float16,float16,0,10.723129272460938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,40,4,128,1,fp8,fp8,0,18.50465087890625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,40,8,128,1,float16,fp8,0,19.4654296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,40,8,128,1,fp8,fp8,0,19.143125915527342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,1,128,1,float16,fp8,0,9.291088104248047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,40,8,128,1,float16,float16,0,22.183192443847656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,1,128,1,fp8,fp8,0,8.995044708251953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,2,128,1,float16,float16,0,10.86324462890625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,2,128,1,fp8,fp8,0,9.185912322998046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,2,128,1,float16,fp8,0,9.412782287597656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,4,128,1,float16,fp8,0,9.308262634277344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,4,128,1,fp8,fp8,0,9.369884490966797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,4,128,1,float16,float16,0,11.276006317138672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,8,128,1,float16,fp8,0,9.174062347412109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,40,128,1,float16,float16,0,6.247574234008789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,8,128,1,float16,float16,0,11.324187469482421
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,40,128,1,float16,fp8,0,4.650204849243164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,40,8,128,1,fp8,fp8,0,9.379590606689453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,40,128,1,fp8,fp8,0,4.753919982910157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,1,128,1,float16,fp8,0,4.667895889282226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,1,128,1,fp8,fp8,0,4.610623931884765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,1,128,1,float16,float16,0,5.901895904541016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,2,128,1,float16,float16,0,5.5657390594482425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,2,128,1,fp8,fp8,0,4.538689422607422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,4,128,1,float16,fp8,0,4.709179306030274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,4,128,1,float16,float16,0,5.527374267578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,4,128,1,fp8,fp8,0,4.608652877807617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,40,128,1,float16,float16,0,2.6063663482666017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,2,128,1,float16,fp8,0,4.628275299072266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,8,128,1,float16,fp8,0,4.931480026245117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,40,128,1,float16,fp8,0,2.4380752563476564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,40,128,1,fp8,fp8,0,2.598860740661621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,8,128,1,fp8,fp8,0,4.6826526641845705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,1,128,1,float16,float16,0,2.8044639587402345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,1,128,1,float16,fp8,0,2.554622459411621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,1,128,1,fp8,fp8,0,2.2283632278442385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,2,128,1,float16,float16,0,2.4735504150390626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,2,128,1,float16,fp8,0,2.2641664505004884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,2,128,1,fp8,fp8,0,2.2628063201904296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,40,8,128,1,float16,float16,0,5.6380046844482425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,4,128,1,float16,fp8,0,2.2743408203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,4,128,1,float16,float16,0,2.7606592178344727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,4,128,1,fp8,fp8,0,2.2715503692626955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,8,128,1,float16,fp8,0,2.2656368255615233
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,8,128,1,fp8,fp8,0,2.267624092102051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,40,8,128,1,float16,float16,0,3.0779151916503906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,40,1,128,1,float16,fp8,0,12.738873291015626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,40,1,128,1,fp8,fp8,0,13.152113342285157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,40,1,128,1,float16,float16,0,15.004290771484374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,40,2,128,1,fp8,fp8,0,12.916069030761719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,40,2,128,1,float16,float16,0,15.183515930175782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,40,4,128,1,float16,fp8,0,12.959312438964844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,40,4,128,1,float16,float16,0,15.973043823242188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,40,2,128,1,float16,fp8,0,12.832620239257812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,40,128,1,float16,fp8,0,6.737850952148437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,40,128,1,float16,float16,0,7.976783752441406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,40,128,1,fp8,fp8,0,6.778604888916016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,1,128,1,float16,float16,0,7.374823760986328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,40,4,128,1,fp8,fp8,0,13.3685791015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,40,8,128,1,float16,fp8,0,12.79189453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,40,8,128,1,fp8,fp8,0,13.557492065429688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,1,128,1,float16,fp8,0,6.451152038574219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,40,8,128,1,float16,float16,0,16.106172180175783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,1,128,1,fp8,fp8,0,6.894929504394531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,2,128,1,float16,fp8,0,6.456121826171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,2,128,1,fp8,fp8,0,6.507513427734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,2,128,1,float16,float16,0,7.840198516845703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,4,128,1,float16,fp8,0,6.61268310546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,4,128,1,float16,float16,0,8.380876922607422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,4,128,1,fp8,fp8,0,6.565737915039063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,40,128,1,float16,fp8,0,3.4901054382324217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,40,128,1,float16,float16,0,4.091683197021484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,8,128,1,float16,fp8,0,6.560817718505859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,8,128,1,float16,float16,0,7.93920669555664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,40,128,1,fp8,fp8,0,3.3601215362548826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,40,8,128,1,fp8,fp8,0,6.419046020507812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,1,128,1,float16,fp8,0,3.4599025726318358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,1,128,1,float16,float16,0,3.75382080078125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,1,128,1,fp8,fp8,0,3.1822847366333007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,2,128,1,float16,fp8,0,3.173931121826172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,2,128,1,float16,float16,0,3.7745391845703127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,2,128,1,fp8,fp8,0,3.35473747253418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,4,128,1,float16,fp8,0,3.1886335372924806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,4,128,1,float16,float16,0,3.81866569519043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,4,128,1,fp8,fp8,0,3.306294250488281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,40,128,1,float16,float16,0,1.9902944564819336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,40,128,1,float16,fp8,0,1.6910432815551757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,8,128,1,float16,float16,0,3.795091247558594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,40,128,1,fp8,fp8,0,1.6853103637695312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,1,128,1,float16,float16,0,1.8744991302490235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,1,128,1,float16,fp8,0,1.6292320251464845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,1,128,1,fp8,fp8,0,1.6761760711669922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,2,128,1,float16,float16,0,1.9388303756713867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,2,128,1,float16,fp8,0,1.7157968521118163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,2,128,1,fp8,fp8,0,1.5730272293090821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,4,128,1,float16,fp8,0,1.6181343078613282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,4,128,1,float16,float16,0,1.8667552947998047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,4,128,1,fp8,fp8,0,1.6022031784057618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,8,128,1,float16,fp8,0,3.27628173828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,8,128,1,float16,fp8,0,1.624190330505371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,8,128,1,float16,float16,0,1.941979217529297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,40,8,128,1,fp8,fp8,0,1.5847663879394531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,40,8,128,1,fp8,fp8,0,3.216483306884766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,40,1,128,1,fp8,fp8,0,16.855265808105468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,40,2,128,1,float16,fp8,0,16.56696319580078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,40,4,128,1,float16,fp8,0,16.635922241210938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,40,2,128,1,fp8,fp8,0,17.01420440673828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,40,1,128,1,float16,float16,0,20.095506286621095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,40,2,128,1,float16,float16,0,20.05771484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,40,1,128,1,float16,fp8,0,16.846824645996094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,40,4,128,1,float16,float16,0,20.70824279785156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,40,128,1,float16,fp8,0,8.856543731689452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,40,128,1,fp8,fp8,0,8.793561553955078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,40,128,1,float16,float16,0,11.339864349365234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,1,128,1,float16,float16,0,9.883946990966797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,40,4,128,1,fp8,fp8,0,17.01183319091797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,40,8,128,1,float16,fp8,0,17.057902526855468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,40,8,128,1,fp8,fp8,0,17.552313232421874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,40,8,128,1,float16,float16,0,20.899264526367187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,1,128,1,fp8,fp8,0,8.326142120361329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,1,128,1,float16,fp8,0,8.950492858886719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,2,128,1,float16,fp8,0,8.325804901123046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,2,128,1,float16,float16,0,10.137506866455078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,2,128,1,fp8,fp8,0,8.397169494628907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,4,128,1,float16,fp8,0,8.620527648925782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,4,128,1,float16,float16,0,10.436102294921875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,4,128,1,fp8,fp8,0,8.288988494873047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,40,128,1,float16,float16,0,5.361756896972656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,40,128,1,float16,fp8,0,4.383659362792969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,8,128,1,float16,fp8,0,8.526558685302735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,8,128,1,float16,float16,0,9.990534210205078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,40,8,128,1,fp8,fp8,0,8.594537353515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,40,128,1,fp8,fp8,0,4.446190261840821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,1,128,1,float16,float16,0,4.87071533203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,1,128,1,float16,fp8,0,4.095523071289063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,1,128,1,fp8,fp8,0,4.210710525512695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,2,128,1,float16,float16,0,5.1446880340576175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,2,128,1,float16,fp8,0,4.260931015014648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,2,128,1,fp8,fp8,0,4.2076255798339846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,4,128,1,float16,float16,0,5.157880020141602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,4,128,1,fp8,fp8,0,4.132043075561524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,40,128,1,float16,float16,0,2.6154863357543947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,8,128,1,float16,float16,0,5.069537734985351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,40,128,1,fp8,fp8,0,2.155422401428223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,8,128,1,float16,fp8,0,4.290611267089844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,8,128,1,fp8,fp8,0,4.57702407836914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,1,128,1,float16,fp8,0,2.2802671432495116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,40,4,128,1,float16,fp8,0,4.2508384704589846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,2,128,1,float16,fp8,0,2.063750457763672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,2,128,1,float16,float16,0,2.356025505065918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,2,128,1,fp8,fp8,0,2.227265548706055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,40,128,1,float16,fp8,0,2.357592010498047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,4,128,1,float16,float16,0,2.32983512878418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,4,128,1,float16,fp8,0,2.058700752258301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,1,128,1,float16,float16,0,2.1619199752807616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,4,128,1,fp8,fp8,0,2.0670543670654298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,8,128,1,float16,float16,0,2.5066015243530275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,8,128,1,float16,fp8,0,2.1008159637451174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,40,128,1,fp8,fp8,0,1.0824432373046875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,40,128,1,float16,fp8,0,1.0823887825012206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,8,128,1,fp8,fp8,0,2.0582223892211915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,40,1,128,1,fp8,fp8,0,2.1914255142211916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,1,128,1,float16,fp8,0,1.1664112091064454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,2,128,1,float16,float16,0,1.086350440979004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,2,128,1,float16,fp8,0,1.028551959991455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,2,128,1,fp8,fp8,0,1.0327936172485352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,4,128,1,float16,float16,0,1.1039487838745117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,4,128,1,float16,fp8,0,1.0949343681335448
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,4,128,1,fp8,fp8,0,1.142734432220459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,8,128,1,float16,float16,0,1.11953763961792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,8,128,1,float16,fp8,0,1.033456039428711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,8,128,1,fp8,fp8,0,1.1491040229797362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,40,128,1,float16,float16,0,1.2807279586791993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,1,128,1,float16,float16,0,1.1039055824279784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,40,1,128,1,fp8,fp8,0,1.0485679626464843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,40,1,128,1,float16,fp8,0,9.477591705322265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,40,1,128,1,fp8,fp8,0,9.5342529296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,40,2,128,1,float16,fp8,0,9.750911712646484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,40,1,128,1,float16,float16,0,11.488572692871093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,40,2,128,1,float16,float16,0,11.576033782958984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,40,4,128,1,float16,fp8,0,9.687643432617188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,40,4,128,1,float16,float16,0,11.339103698730469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,40,2,128,1,fp8,fp8,0,9.635100555419921
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,40,128,1,float16,fp8,0,5.1080272674560545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,40,128,1,fp8,fp8,0,5.14901123046875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,40,128,1,float16,float16,0,6.204489517211914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,40,4,128,1,fp8,fp8,0,9.562548828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,1,128,1,float16,float16,0,5.438638305664062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,40,8,128,1,float16,fp8,0,9.854081726074218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,40,8,128,1,fp8,fp8,0,9.84882049560547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,40,8,128,1,float16,float16,0,11.52172622680664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,1,128,1,float16,fp8,0,4.800455856323242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,1,128,1,fp8,fp8,0,4.774903869628906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,2,128,1,float16,float16,0,5.709609603881836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,2,128,1,float16,fp8,0,4.747772979736328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,2,128,1,fp8,fp8,0,4.836585617065429
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,4,128,1,float16,float16,0,5.803612899780274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,4,128,1,float16,fp8,0,5.177238464355469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,4,128,1,fp8,fp8,0,5.005028915405274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,40,128,1,float16,float16,0,2.792555236816406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,8,128,1,float16,float16,0,5.692036819458008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,8,128,1,float16,fp8,0,4.835107040405274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,40,128,1,fp8,fp8,0,2.6698671340942384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,1,128,1,float16,float16,0,2.7787071228027345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,40,8,128,1,fp8,fp8,0,5.063496017456055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,1,128,1,fp8,fp8,0,2.6258480072021486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,2,128,1,float16,fp8,0,2.399286460876465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,2,128,1,float16,float16,0,2.813836860656738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,2,128,1,fp8,fp8,0,2.409614372253418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,40,128,1,float16,fp8,0,2.539684867858887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,4,128,1,float16,fp8,0,2.5761632919311523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,4,128,1,float16,float16,0,2.981287956237793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,4,128,1,fp8,fp8,0,2.4675600051879885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,1,128,1,float16,fp8,0,2.406488037109375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,40,128,1,float16,float16,0,1.4135343551635742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,8,128,1,float16,float16,0,2.839116859436035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,8,128,1,float16,fp8,0,2.5593839645385743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,40,128,1,float16,fp8,0,1.3201807975769042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,40,128,1,fp8,fp8,0,1.4865568161010743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,1,128,1,float16,float16,0,1.3055968284606934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,40,8,128,1,fp8,fp8,0,2.5905216217041014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,1,128,1,float16,fp8,0,1.1913984298706055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,1,128,1,fp8,fp8,0,1.2825776100158692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,2,128,1,fp8,fp8,0,1.2018848419189454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,2,128,1,float16,float16,0,1.5847104072570801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,4,128,1,float16,float16,0,1.3272080421447754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,4,128,1,float16,fp8,0,1.4299551963806152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,8,128,1,float16,float16,0,1.3118720054626465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,8,128,1,float16,fp8,0,1.2657456398010254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,40,128,1,float16,fp8,0,0.6934080123901367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,40,128,1,fp8,fp8,0,0.706822395324707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,1,128,1,float16,float16,0,0.6658080101013184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,1,128,1,float16,fp8,0,0.5987343788146973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,1,128,1,fp8,fp8,0,0.6333343982696533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,2,128,1,float16,float16,0,0.687992000579834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,2,128,1,float16,fp8,0,0.6001552104949951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,2,128,1,fp8,fp8,0,0.6176976203918457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,4,128,1,float16,float16,0,0.6861055850982666
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,4,128,1,float16,fp8,0,0.602953577041626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,2,128,1,float16,fp8,0,1.2339056015014649
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,4,128,1,fp8,fp8,0,0.6078735828399658
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,8,128,1,float16,float16,0,0.6927184104919434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,8,128,1,float16,fp8,0,0.6077824115753174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,8,128,1,fp8,fp8,0,0.606115198135376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,40,40,128,1,float16,float16,0,0.876318359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,8,128,1,fp8,fp8,0,1.2142047882080078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,40,4,128,1,fp8,fp8,0,1.2589344024658202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,40,1,128,1,fp8,fp8,0,8.903227233886719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,40,2,128,1,float16,fp8,0,8.941524505615234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,40,1,128,1,float16,float16,0,10.531156921386719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,40,2,128,1,fp8,fp8,0,9.014256286621094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,40,2,128,1,float16,float16,0,10.603498840332032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,40,1,128,1,float16,fp8,0,8.965350341796874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,40,4,128,1,float16,fp8,0,9.09464340209961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,40,4,128,1,float16,float16,0,10.67825927734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,40,128,1,float16,float16,0,5.8449455261230465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,40,128,1,float16,fp8,0,4.871926498413086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,40,4,128,1,fp8,fp8,0,9.060486602783204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,40,128,1,fp8,fp8,0,4.956407928466797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,1,128,1,float16,float16,0,5.295091247558593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,40,8,128,1,float16,fp8,0,9.092774200439454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,40,8,128,1,fp8,fp8,0,9.219779205322265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,40,8,128,1,float16,float16,0,10.6856689453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,1,128,1,float16,fp8,0,4.564507293701172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,1,128,1,fp8,fp8,0,4.448976135253906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,2,128,1,float16,fp8,0,4.510015869140625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,2,128,1,float16,float16,0,5.176505661010742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,2,128,1,fp8,fp8,0,4.547956848144532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,4,128,1,float16,fp8,0,4.522531127929687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,4,128,1,float16,float16,0,5.217862319946289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,4,128,1,fp8,fp8,0,4.592587280273437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,40,128,1,float16,float16,0,3.0701488494873046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,40,128,1,fp8,fp8,0,2.4501615524291993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,8,128,1,float16,float16,0,5.359758377075195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,8,128,1,float16,fp8,0,4.558414459228516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,1,128,1,float16,float16,0,2.549246406555176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,40,8,128,1,fp8,fp8,0,4.5856575012207035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,1,128,1,fp8,fp8,0,2.4194208145141602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,2,128,1,float16,fp8,0,2.355681610107422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,2,128,1,float16,float16,0,2.780745506286621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,4,128,1,float16,float16,0,2.6834287643432617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,4,128,1,float16,fp8,0,2.2583616256713865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,40,128,1,float16,fp8,0,2.4261615753173826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,1,128,1,float16,fp8,0,2.232899284362793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,40,128,1,float16,fp8,0,1.2167632102966308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,8,128,1,float16,float16,0,2.6666431427001953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,8,128,1,float16,fp8,0,2.4916767120361327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,8,128,1,fp8,fp8,0,2.706318473815918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,1,128,1,float16,float16,0,1.1935664176940919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,2,128,1,fp8,fp8,0,2.2534704208374023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,1,128,1,fp8,fp8,0,1.1289567947387695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,1,128,1,float16,fp8,0,1.4102479934692382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,2,128,1,float16,float16,0,1.1802864074707031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,2,128,1,float16,fp8,0,1.2591631889343262
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,2,128,1,fp8,fp8,0,1.1228495597839356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,4,128,1,float16,float16,0,1.2456607818603516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,4,128,1,float16,fp8,0,1.198412799835205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,40,4,128,1,fp8,fp8,0,2.244828796386719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,4,128,1,fp8,fp8,0,1.1954416275024413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,8,128,1,float16,float16,0,1.3133071899414062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,8,128,1,float16,fp8,0,1.1359807968139648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,40,128,1,float16,fp8,0,0.6172688007354736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,40,128,1,fp8,fp8,0,0.6276512145996094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,8,128,1,fp8,fp8,0,1.372753620147705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,1,128,1,float16,float16,0,0.641428804397583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,1,128,1,fp8,fp8,0,0.5668208122253418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,2,128,1,float16,float16,0,0.5913983821868897
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,2,128,1,float16,fp8,0,0.6314144134521484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,2,128,1,fp8,fp8,0,0.5995039939880371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,4,128,1,float16,float16,0,0.5989120006561279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,4,128,1,float16,fp8,0,0.5701727867126465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,4,128,1,fp8,fp8,0,0.6358911991119385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,8,128,1,float16,fp8,0,0.5747839927673339
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,8,128,1,fp8,fp8,0,0.5750495910644531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,40,128,1,float16,float16,0,0.3678960084915161
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,40,128,1,float16,fp8,0,0.31581759452819824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,40,128,1,fp8,fp8,0,0.3164959907531738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,1,128,1,float16,float16,0,0.3109071969985962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,40,128,1,float16,float16,0,1.3718576431274414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,1,128,1,fp8,fp8,0,0.2876496076583862
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,2,128,1,float16,float16,0,0.31227519512176516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,40,128,1,float16,float16,0,0.6836448192596436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,2,128,1,float16,fp8,0,0.28638079166412356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,2,128,1,fp8,fp8,0,0.28860960006713865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,4,128,1,float16,float16,0,0.3109231948852539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,4,128,1,float16,fp8,0,0.2898704051971436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,4,128,1,fp8,fp8,0,0.2883471965789795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,8,128,1,float16,float16,0,0.3066720008850098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,40,40,128,1,fp8,fp8,0,1.219324779510498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,8,128,1,fp8,fp8,0,0.29237439632415774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,8,128,1,float16,float16,0,0.620630407333374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,1,128,1,float16,fp8,0,0.2868560075759888
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,40,1,128,1,float16,fp8,0,0.5638527870178223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,40,1,128,1,float16,fp8,0,5.260217666625977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,40,1,128,1,fp8,fp8,0,5.257640075683594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,40,1,128,1,float16,float16,0,6.077297592163086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,40,8,128,1,float16,fp8,0,0.29436159133911133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,40,2,128,1,float16,float16,0,6.129987335205078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,40,2,128,1,float16,fp8,0,5.227928161621094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,40,2,128,1,fp8,fp8,0,5.288715362548828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,40,4,128,1,float16,fp8,0,5.3182415008544925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,40,4,128,1,float16,float16,0,6.147739028930664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,40,128,1,float16,float16,0,3.3831310272216797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,40,128,1,float16,fp8,0,2.942091178894043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,40,8,128,1,float16,float16,0,6.127910232543945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,40,8,128,1,float16,fp8,0,5.417728042602539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,40,8,128,1,fp8,fp8,0,5.476299285888672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,40,128,1,fp8,fp8,0,2.9426015853881835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,1,128,1,float16,fp8,0,2.7335023880004883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,1,128,1,fp8,fp8,0,2.6452400207519533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,2,128,1,float16,fp8,0,2.6612911224365234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,2,128,1,fp8,fp8,0,2.6407743453979493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,4,128,1,float16,float16,0,2.873489570617676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,4,128,1,float16,fp8,0,2.6621456146240234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,4,128,1,fp8,fp8,0,2.66378231048584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,1,128,1,float16,float16,0,2.9941776275634764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,40,4,128,1,fp8,fp8,0,5.404361724853516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,8,128,1,float16,float16,0,2.9683807373046873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,40,128,1,float16,float16,0,1.6710704803466796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,40,128,1,float16,fp8,0,1.4747535705566406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,40,128,1,fp8,fp8,0,1.4740351676940917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,8,128,1,fp8,fp8,0,2.795846366882324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,1,128,1,float16,float16,0,1.378438377380371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,1,128,1,float16,fp8,0,1.5755375862121581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,1,128,1,fp8,fp8,0,1.4316271781921386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,2,128,1,float16,float16,0,1.4043264389038086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,2,128,1,float16,fp8,0,1.3270336151123048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,2,128,1,fp8,fp8,0,1.3200575828552246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,4,128,1,float16,float16,0,1.5045040130615235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,4,128,1,float16,fp8,0,1.3678447723388671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,4,128,1,fp8,fp8,0,1.3621295928955077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,8,128,1,float16,fp8,0,2.6896976470947265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,8,128,1,float16,float16,0,1.4368127822875976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,8,128,1,float16,fp8,0,1.354047966003418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,40,128,1,float16,float16,0,0.8704671859741211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,40,8,128,1,fp8,fp8,0,1.343945598602295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,40,128,1,fp8,fp8,0,0.7454607963562012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,1,128,1,float16,float16,0,0.7020495891571045
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,1,128,1,fp8,fp8,0,0.6703375816345215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,2,128,1,float16,fp8,0,0.6661632061004639
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,2,128,1,fp8,fp8,0,0.6705152034759522
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,4,128,1,float16,float16,0,0.7126351833343506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,4,128,1,float16,fp8,0,0.6777567863464355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,4,128,1,fp8,fp8,0,0.671724796295166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,8,128,1,float16,float16,0,0.7148767948150635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,8,128,1,float16,fp8,0,0.6807744026184082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,40,128,1,float16,float16,0,0.42197442054748535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,8,128,1,fp8,fp8,0,0.685268783569336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,40,128,1,float16,fp8,0,0.40926079750061034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,40,128,1,fp8,fp8,0,0.37683041095733644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,1,128,1,float16,float16,0,0.3539344072341919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,1,128,1,float16,fp8,0,0.34158399105072024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,1,128,1,fp8,fp8,0,0.3639280080795288
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,2,128,1,float16,float16,0,0.360316801071167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,2,128,1,float16,fp8,0,0.340118408203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,2,128,1,fp8,fp8,0,0.34198238849639895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,4,128,1,float16,float16,0,0.36911520957946775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,4,128,1,float16,fp8,0,0.3410943984985352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,4,128,1,fp8,fp8,0,0.3423135995864868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,8,128,1,float16,float16,0,0.36267518997192383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,8,128,1,float16,fp8,0,0.3567823886871338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,40,8,128,1,fp8,fp8,0,0.3455600023269653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,40,128,1,float16,fp8,0,0.19582560062408447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,1,128,1,float16,fp8,0,0.74475998878479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,40,128,1,fp8,fp8,0,0.19958399534225463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,1,128,1,float16,float16,0,0.18592159748077391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,1,128,1,fp8,fp8,0,0.17609920501708984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,2,128,1,float16,float16,0,0.1851904034614563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,2,128,1,float16,float16,0,0.7401919841766358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,2,128,1,float16,fp8,0,0.1748319983482361
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,2,128,1,fp8,fp8,0,0.1766495943069458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,4,128,1,float16,float16,0,0.18695839643478393
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,4,128,1,float16,fp8,0,0.17764639854431152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,8,128,1,float16,float16,0,0.19013600349426268
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,8,128,1,float16,fp8,0,0.18069119453430177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,8,128,1,fp8,fp8,0,0.18067840337753296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,40,40,128,1,float16,fp8,0,0.7495359897613525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,40,2,128,1,float16,float16,0,2.998299217224121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,40,128,1,float16,float16,0,0.2214479923248291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,1,128,1,float16,fp8,0,0.17633600234985353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,40,4,128,1,fp8,fp8,0,0.17610559463500977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,40,1,128,1,float16,fp8,0,5.156320190429687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,40,1,128,1,float16,float16,0,5.722828674316406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,40,2,128,1,float16,fp8,0,5.128193664550781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,40,2,128,1,fp8,fp8,0,5.148102569580078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,40,2,128,1,float16,float16,0,5.829467010498047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,40,4,128,1,float16,float16,0,5.797851181030273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,40,4,128,1,float16,fp8,0,5.203839874267578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,40,1,128,1,fp8,fp8,0,5.133100891113282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,40,4,128,1,fp8,fp8,0,5.2180126190185545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,40,8,128,1,float16,float16,0,5.937630462646484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,40,128,1,float16,float16,0,3.5152751922607424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,1,128,1,float16,float16,0,2.7832752227783204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,40,128,1,float16,fp8,0,3.0475040435791017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,40,128,1,fp8,fp8,0,2.976919937133789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,40,8,128,1,float16,fp8,0,5.289257431030274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,40,8,128,1,fp8,fp8,0,5.282918548583984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,1,128,1,float16,fp8,0,2.6625871658325195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,1,128,1,fp8,fp8,0,2.7915264129638673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,2,128,1,float16,fp8,0,2.596124839782715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,2,128,1,fp8,fp8,0,2.594700813293457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,2,128,1,float16,float16,0,2.9444976806640626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,4,128,1,float16,float16,0,2.8502063751220703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,4,128,1,float16,fp8,0,2.6047183990478517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,40,128,1,float16,float16,0,1.696963119506836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,4,128,1,fp8,fp8,0,2.619843292236328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,8,128,1,float16,fp8,0,2.667849540710449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,8,128,1,float16,float16,0,2.8962591171264647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,1,128,1,float16,float16,0,1.4522591590881349
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,1,128,1,float16,fp8,0,1.303673553466797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,1,128,1,fp8,fp8,0,1.3039792060852051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,2,128,1,float16,float16,0,1.4727215766906738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,2,128,1,float16,fp8,0,1.2931376457214356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,2,128,1,fp8,fp8,0,1.3837823867797852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,4,128,1,float16,float16,0,1.39028959274292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,4,128,1,float16,fp8,0,1.317251205444336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,4,128,1,fp8,fp8,0,1.4184911727905274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,8,128,1,float16,float16,0,1.3837519645690919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,40,8,128,1,fp8,fp8,0,2.6617759704589843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,40,128,1,float16,fp8,0,1.5961407661437987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,40,128,1,float16,float16,0,0.828651237487793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,8,128,1,fp8,fp8,0,1.3978207588195801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,40,128,1,float16,fp8,0,0.7616288185119628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,40,128,1,fp8,fp8,0,0.8137359619140625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,1,128,1,float16,float16,0,0.6781648159027099
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,1,128,1,float16,fp8,0,0.6576032161712646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,1,128,1,fp8,fp8,0,0.6547728061676026
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,2,128,1,float16,float16,0,0.6843696117401123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,2,128,1,float16,fp8,0,0.6587503910064697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,2,128,1,fp8,fp8,0,0.6732495784759521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,4,128,1,float16,float16,0,0.6896431922912598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,4,128,1,float16,fp8,0,0.6654352188110352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,4,128,1,fp8,fp8,0,0.6598495960235595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,40,128,1,float16,fp8,0,0.38676159381866454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,40,128,1,float16,float16,0,0.426961612701416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,8,128,1,float16,float16,0,0.7068304061889649
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,1,128,1,float16,float16,0,0.3429903984069824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,1,128,1,fp8,fp8,0,0.33729119300842286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,2,128,1,float16,float16,0,0.3451103925704956
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,2,128,1,float16,fp8,0,0.33597760200500487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,40,128,1,fp8,fp8,0,1.4881216049194337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,2,128,1,fp8,fp8,0,0.33617119789123534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,40,8,128,1,float16,fp8,0,1.3473183631896972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,4,128,1,float16,float16,0,0.3500207901000977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,4,128,1,fp8,fp8,0,0.338372802734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,8,128,1,float16,float16,0,0.356548810005188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,40,128,1,float16,float16,0,0.21933441162109374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,40,128,1,float16,fp8,0,0.1997167944908142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,40,128,1,fp8,fp8,0,0.199726402759552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,1,128,1,float16,float16,0,0.1770848035812378
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,1,128,1,float16,fp8,0,0.17161920070648193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,40,128,1,fp8,fp8,0,0.38392159938812254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,8,128,1,float16,fp8,0,0.6712783813476563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,1,128,1,fp8,fp8,0,0.17154079675674438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,2,128,1,float16,float16,0,0.1776255965232849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,2,128,1,float16,fp8,0,0.17296479940414428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,2,128,1,fp8,fp8,0,0.17248320579528809
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,4,128,1,float16,float16,0,0.1795680046081543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,4,128,1,float16,fp8,0,0.1746896028518677
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,4,128,1,fp8,fp8,0,0.17396960258483887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,8,128,1,float16,float16,0,0.18418079614639282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,8,128,1,float16,fp8,0,0.17764960527420043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,40,8,128,1,fp8,fp8,0,0.1770751953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,40,128,1,float16,float16,0,0.12298719882965088
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,40,128,1,float16,fp8,0,0.11058399677276612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,40,128,1,fp8,fp8,0,0.11042239665985107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,1,128,1,float16,fp8,0,0.0883903980255127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,1,128,1,fp8,fp8,0,0.08888959884643555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,2,128,1,float16,float16,0,0.09472799897193909
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,2,128,1,float16,fp8,0,0.0894864022731781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,4,128,1,float16,fp8,0,0.33699679374694824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,8,128,1,float16,fp8,0,0.34400479793548583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,4,128,1,float16,float16,0,0.09566239714622497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,4,128,1,float16,fp8,0,0.08980960249900818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,4,128,1,fp8,fp8,0,0.09019839763641357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,8,128,1,float16,float16,0,0.0987600028514862
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,8,128,1,float16,fp8,0,0.0922160029411316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,8,128,1,fp8,fp8,0,0.09255999922752381
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,40,8,128,1,fp8,fp8,0,0.6862559795379639
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,1,128,1,float16,fp8,0,0.33198559284210205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,1,128,1,float16,float16,0,0.09242399930953979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,40,2,128,1,fp8,fp8,0,0.08878560066223144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,40,1,128,1,float16,fp8,0,3.1718576431274412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,40,1,128,1,fp8,fp8,0,3.1912864685058593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,40,1,128,1,float16,float16,0,3.392873764038086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,40,8,128,1,fp8,fp8,0,0.34475040435791016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,40,2,128,1,float16,float16,0,3.295033645629883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,40,2,128,1,float16,fp8,0,3.153004837036133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,40,2,128,1,fp8,fp8,0,3.1658239364624023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,40,4,128,1,float16,float16,0,3.297119903564453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,40,4,128,1,float16,fp8,0,3.1993215560913084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,40,8,128,1,float16,fp8,0,3.275246429443359
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,40,8,128,1,float16,float16,0,3.7012481689453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,40,128,1,float16,float16,0,2.1424528121948243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,40,128,1,float16,fp8,0,1.8913471221923828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,40,8,128,1,fp8,fp8,0,3.346004867553711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,40,128,1,fp8,fp8,0,1.8833248138427734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,1,128,1,float16,float16,0,1.620591926574707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,1,128,1,float16,fp8,0,1.7075807571411132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,1,128,1,fp8,fp8,0,1.6698720932006836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,2,128,1,float16,float16,0,1.6194639205932617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,2,128,1,float16,fp8,0,1.6012544631958008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,40,4,128,1,fp8,fp8,0,3.213332748413086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,2,128,1,fp8,fp8,0,1.5847087860107423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,4,128,1,float16,float16,0,1.7011808395385741
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,4,128,1,float16,fp8,0,1.6414640426635743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,4,128,1,fp8,fp8,0,1.6151487350463867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,40,128,1,float16,float16,0,1.0481696128845215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,40,128,1,float16,fp8,0,1.0138671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,40,128,1,fp8,fp8,0,0.953335952758789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,1,128,1,float16,float16,0,0.822441577911377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,1,128,1,float16,fp8,0,0.8149359703063965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,1,128,1,fp8,fp8,0,0.823465633392334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,2,128,1,float16,fp8,0,0.8034527778625489
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,2,128,1,fp8,fp8,0,0.8086175918579102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,4,128,1,float16,float16,0,0.8387616157531739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,4,128,1,fp8,fp8,0,0.8186575889587402
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,8,128,1,float16,float16,0,0.887399959564209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,8,128,1,float16,float16,0,1.7195695877075194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,8,128,1,float16,fp8,0,1.6538272857666017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,40,8,128,1,fp8,fp8,0,1.6440496444702148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,8,128,1,fp8,fp8,0,0.8653663635253906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,40,128,1,float16,fp8,0,0.4900320053100586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,40,128,1,float16,float16,0,0.5350863933563232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,40,128,1,fp8,fp8,0,0.4905216217041016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,1,128,1,float16,float16,0,0.41535201072692873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,1,128,1,float16,fp8,0,0.4129039764404297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,2,128,1,float16,float16,0,0.8216912269592285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,1,128,1,fp8,fp8,0,0.41017441749572753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,2,128,1,float16,float16,0,0.42168641090393066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,2,128,1,float16,fp8,0,0.4116847991943359
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,2,128,1,fp8,fp8,0,0.4090864181518555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,4,128,1,float16,float16,0,0.4256608009338379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,4,128,1,float16,fp8,0,0.41295838356018066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,4,128,1,fp8,fp8,0,0.41655521392822265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,8,128,1,float16,float16,0,0.43827037811279296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,8,128,1,float16,fp8,0,0.42559361457824707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,40,8,128,1,fp8,fp8,0,0.422430419921875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,40,128,1,float16,fp8,0,0.25237441062927246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,4,128,1,float16,fp8,0,0.813167953491211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,40,128,1,fp8,fp8,0,0.25405919551849365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,1,128,1,float16,float16,0,0.2149951934814453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,1,128,1,float16,fp8,0,0.21196479797363282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,1,128,1,fp8,fp8,0,0.21017439365386964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,2,128,1,float16,float16,0,0.21480159759521483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,2,128,1,float16,fp8,0,0.2111583948135376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,2,128,1,fp8,fp8,0,0.21265439987182616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,4,128,1,float16,float16,0,0.21984639167785644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,4,128,1,float16,fp8,0,0.21461920738220214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,8,128,1,float16,float16,0,0.2243056058883667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,8,128,1,float16,fp8,0,0.21755199432373046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,8,128,1,fp8,fp8,0,0.21921279430389404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,40,128,1,float16,fp8,0,0.13602559566497802
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,40,128,1,fp8,fp8,0,0.13511680364608764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,1,128,1,float16,float16,0,0.11367839574813843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,1,128,1,float16,fp8,0,0.11137440204620361
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,1,128,1,fp8,fp8,0,0.11129440069198608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,2,128,1,float16,float16,0,0.11416959762573242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,2,128,1,float16,fp8,0,0.1106768012046814
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,2,128,1,fp8,fp8,0,0.11174399852752685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,4,128,1,float16,fp8,0,0.11309280395507812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,8,128,1,float16,float16,0,0.119651198387146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,8,128,1,float16,fp8,0,0.11455999612808228
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,40,8,128,1,float16,fp8,0,0.8371439933776855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,40,128,1,float16,float16,0,0.08313440084457398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,40,128,1,float16,fp8,0,0.07742080092430115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,1,128,1,float16,float16,0,0.05985280275344849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,1,128,1,float16,fp8,0,0.05870400071144104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,1,128,1,fp8,fp8,0,0.05869920253753662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,2,128,1,float16,float16,0,0.05995519757270813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,2,128,1,float16,fp8,0,0.05865439772605896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,2,128,1,fp8,fp8,0,0.058697599172592166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,4,128,1,float16,float16,0,0.060496002435684204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,4,128,1,float16,fp8,0,0.05903519988059998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,40,128,1,float16,float16,0,0.2751712083816528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,8,128,1,float16,float16,0,0.06406559944152831
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,8,128,1,float16,fp8,0,0.060102397203445436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,8,128,1,fp8,fp8,0,0.05981919765472412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,40,4,128,1,fp8,fp8,0,0.21353600025177003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,40,128,1,float16,float16,0,0.1460479974746704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,4,128,1,float16,float16,0,0.11573280096054077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,4,128,1,fp8,fp8,0,0.11217600107192993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,40,8,128,1,fp8,fp8,0,0.11527040004730224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,40,128,1,fp8,fp8,0,0.07725759744644164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,40,4,128,1,fp8,fp8,0,0.05894880294799805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,40,1,128,1,float16,float16,0,3.3936927795410154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,40,1,128,1,float16,fp8,0,3.317478561401367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,40,1,128,1,fp8,fp8,0,3.2968208312988283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,40,2,128,1,float16,float16,0,3.383438491821289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,40,2,128,1,float16,fp8,0,3.305156707763672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,40,2,128,1,fp8,fp8,0,3.340427017211914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,40,4,128,1,float16,float16,0,3.441455841064453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,40,4,128,1,float16,fp8,0,3.376278305053711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,40,4,128,1,fp8,fp8,0,3.407747268676758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,40,128,1,float16,float16,0,2.3039871215820313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,40,8,128,1,float16,fp8,0,3.4432209014892576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,40,8,128,1,float16,float16,0,3.695102310180664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,40,128,1,float16,fp8,0,2.1544048309326174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,40,8,128,1,fp8,fp8,0,3.5298465728759765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,40,128,1,fp8,fp8,0,2.0777727127075196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,1,128,1,fp8,fp8,0,1.652337646484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,2,128,1,float16,fp8,0,1.6653600692749024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,2,128,1,fp8,fp8,0,1.6768016815185547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,4,128,1,float16,float16,0,1.7017040252685547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,4,128,1,float16,fp8,0,1.6807968139648437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,4,128,1,fp8,fp8,0,1.6947776794433593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,8,128,1,float16,float16,0,1.7964336395263671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,40,128,1,float16,fp8,0,1.0470208168029784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,1,128,1,float16,float16,0,1.6613967895507813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,1,128,1,float16,fp8,0,1.7487232208251953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,8,128,1,fp8,fp8,0,1.777128028869629
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,2,128,1,float16,float16,0,1.7527231216430663
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,40,128,1,fp8,fp8,0,1.0668288230895997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,1,128,1,float16,float16,0,0.839568042755127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,1,128,1,float16,fp8,0,0.8418704032897949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,1,128,1,fp8,fp8,0,0.846615982055664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,2,128,1,float16,float16,0,0.8516511917114258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,2,128,1,float16,fp8,0,0.8462479591369629
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,2,128,1,fp8,fp8,0,0.8469056129455567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,4,128,1,float16,float16,0,0.8646896362304688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,4,128,1,float16,fp8,0,0.8591039657592774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,4,128,1,fp8,fp8,0,0.8642047882080078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,8,128,1,float16,float16,0,0.9066720008850098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,8,128,1,float16,fp8,0,0.8818623542785644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,40,128,1,float16,float16,0,0.5885727882385254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,8,128,1,fp8,fp8,0,0.880833625793457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,40,128,1,fp8,fp8,0,0.533894395828247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,1,128,1,float16,fp8,0,0.429259204864502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,1,128,1,fp8,fp8,0,0.42663040161132815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,40,40,128,1,float16,float16,0,1.160638427734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,2,128,1,fp8,fp8,0,0.42979679107666013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,4,128,1,float16,float16,0,0.4369152069091797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,40,8,128,1,float16,fp8,0,1.730353546142578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,4,128,1,float16,fp8,0,0.43104162216186526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,4,128,1,fp8,fp8,0,0.43664960861206054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,8,128,1,float16,float16,0,0.45317277908325193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,8,128,1,float16,fp8,0,0.44298720359802246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,40,128,1,float16,float16,0,0.304966402053833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,40,128,1,float16,fp8,0,0.27369439601898193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,40,128,1,fp8,fp8,0,0.27212159633636473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,1,128,1,float16,float16,0,0.2206048011779785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,1,128,1,float16,fp8,0,0.2195296049118042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,1,128,1,fp8,fp8,0,0.21845600605010987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,2,128,1,float16,float16,0,0.22268319129943848
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,2,128,1,float16,fp8,0,0.22035999298095704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,2,128,1,fp8,fp8,0,0.21908159255981446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,4,128,1,float16,float16,0,0.22686879634857177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,4,128,1,float16,fp8,0,0.2235584020614624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,4,128,1,fp8,fp8,0,0.22301919460296632
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,8,128,1,float16,float16,0,0.2347775936126709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,8,128,1,float16,fp8,0,0.2296463966369629
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,40,8,128,1,fp8,fp8,0,0.22901599407196044
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,40,128,1,float16,fp8,0,0.14618560075759887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,40,128,1,fp8,fp8,0,0.145796799659729
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,1,128,1,float16,float16,0,0.11682560443878173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,2,128,1,float16,float16,0,0.43270559310913087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,1,128,1,fp8,fp8,0,0.11645599603652954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,2,128,1,float16,fp8,0,0.43047518730163575
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,2,128,1,float16,fp8,0,0.11715519428253174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,2,128,1,fp8,fp8,0,0.11699199676513672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,4,128,1,float16,float16,0,0.11982560157775879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,4,128,1,float16,fp8,0,0.11872800588607788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,4,128,1,fp8,fp8,0,0.11855360269546508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,8,128,1,float16,float16,0,0.12372159957885742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,8,128,1,float16,fp8,0,0.1215440034866333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,8,128,1,fp8,fp8,0,0.12173600196838379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,40,128,1,float16,float16,0,0.08936319947242737
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,8,128,1,fp8,fp8,0,0.44148478507995603
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,40,128,1,fp8,fp8,0,0.08240799903869629
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,1,128,1,float16,float16,0,0.06272000074386597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,1,128,1,float16,fp8,0,0.061561602354049685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,1,128,1,fp8,fp8,0,0.06133440136909485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,2,128,1,float16,float16,0,0.06447359919548035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,2,128,1,float16,fp8,0,0.06106399893760681
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,4,128,1,float16,float16,0,0.06628640294075012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,4,128,1,float16,fp8,0,0.061934399604797366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,4,128,1,fp8,fp8,0,0.061939197778701785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,8,128,1,float16,float16,0,0.06845279932022094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,8,128,1,float16,fp8,0,0.06412000060081482
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,40,128,1,float16,float16,0,0.16151520013809204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,8,128,1,fp8,fp8,0,0.06468799710273743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,1,128,1,float16,float16,0,0.43317279815673826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,40,128,1,float16,float16,0,0.053827202320098876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,40,128,1,fp8,fp8,0,0.04780640006065369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,1,128,1,float16,fp8,0,0.11628960371017456
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,1,128,1,float16,fp8,0,0.03693119883537292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,1,128,1,fp8,fp8,0,0.036348798871040346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,40,2,128,1,float16,float16,0,0.11806080341339112
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,2,128,1,float16,fp8,0,0.03617919981479645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,2,128,1,fp8,fp8,0,0.03696799874305725
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,4,128,1,float16,float16,0,0.03691200017929077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,4,128,1,float16,fp8,0,0.036664000153541564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,4,128,1,fp8,fp8,0,0.0367680013179779
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,8,128,1,float16,float16,0,0.03744480013847351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,8,128,1,float16,fp8,0,0.036740800738334654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,8,128,1,fp8,fp8,0,0.03675839900970459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,40,128,1,float16,fp8,0,0.08260480165481568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,40,2,128,1,fp8,fp8,0,0.061572802066802976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,40,40,128,1,float16,fp8,0,0.5334144115447998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,40,128,1,float16,fp8,0,0.04760479927062988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,1,128,1,float16,float16,0,0.03660480082035065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,40,1,128,1,float16,float16,0,2.3237775802612304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,40,2,128,1,float16,float16,0,0.03699840009212494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,40,1,128,1,float16,fp8,0,2.3952447891235353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,40,1,128,1,fp8,fp8,0,2.3856143951416016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,40,2,128,1,float16,float16,0,2.3670528411865233
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,40,2,128,1,float16,fp8,0,2.3874160766601564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,40,2,128,1,fp8,fp8,0,2.398823928833008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,40,4,128,1,float16,float16,0,2.4968496322631837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,40,4,128,1,float16,fp8,0,2.456990432739258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,40,4,128,1,fp8,fp8,0,2.443320083618164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,40,8,128,1,float16,float16,0,2.5658639907836913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,1,128,1,float16,float16,0,1.173089599609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,40,128,1,float16,float16,0,1.793459129333496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,40,8,128,1,float16,fp8,0,2.576915168762207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,40,128,1,fp8,fp8,0,1.638804817199707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,40,8,128,1,fp8,fp8,0,2.5470607757568358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,1,128,1,fp8,fp8,0,1.2691568374633788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,2,128,1,float16,float16,0,1.1860688209533692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,2,128,1,float16,fp8,0,1.2096272468566895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,2,128,1,fp8,fp8,0,1.215379238128662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,4,128,1,float16,float16,0,1.220584011077881
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,4,128,1,float16,fp8,0,1.2309023857116699
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,4,128,1,fp8,fp8,0,1.237433624267578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,8,128,1,float16,float16,0,1.290118408203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,40,128,1,float16,float16,0,0.9080623626708985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,8,128,1,float16,fp8,0,1.2876432418823243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,40,128,1,float16,fp8,0,0.8282815933227539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,8,128,1,fp8,fp8,0,1.2933856010437013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,40,128,1,fp8,fp8,0,0.8305855751037597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,1,128,1,float16,float16,0,0.5940767765045166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,1,128,1,float16,fp8,0,0.6104640007019043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,40,128,1,float16,fp8,0,1.6582815170288085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,2,128,1,float16,float16,0,0.601041603088379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,2,128,1,float16,fp8,0,0.6153840065002442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,2,128,1,fp8,fp8,0,0.6111023902893067
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,4,128,1,float16,float16,0,0.6172639846801757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,4,128,1,float16,fp8,0,0.6239952087402344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,4,128,1,fp8,fp8,0,0.628331184387207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,8,128,1,float16,fp8,0,0.6555103778839111
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,8,128,1,fp8,fp8,0,0.6500768184661865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,40,128,1,float16,float16,0,0.46254558563232423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,40,128,1,float16,fp8,0,0.42275681495666506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,40,128,1,fp8,fp8,0,0.4259632110595703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,1,128,1,float16,fp8,0,0.3112207889556885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,1,128,1,fp8,fp8,0,0.3112560033798218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,2,128,1,float16,float16,0,0.30889439582824707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,2,128,1,float16,fp8,0,0.3143183946609497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,2,128,1,fp8,fp8,0,0.31201920509338377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,4,128,1,float16,float16,0,0.31567840576171874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,4,128,1,float16,fp8,0,0.3197135925292969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,4,128,1,fp8,fp8,0,0.32135040760040284
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,8,128,1,float16,float16,0,0.33374719619750975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,8,128,1,float16,fp8,0,0.3316431999206543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,8,128,1,fp8,fp8,0,0.33323519229888915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,40,128,1,float16,fp8,0,0.21980319023132325
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,40,128,1,fp8,fp8,0,0.21929121017456055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,1,128,1,float16,float16,0,0.15934079885482788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,1,128,1,float16,fp8,0,0.16220959424972534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,1,128,1,fp8,fp8,0,0.16194560527801513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,2,128,1,float16,float16,0,0.16148320436477662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,2,128,1,float16,fp8,0,0.16275039911270142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,2,128,1,fp8,fp8,0,0.16226400136947633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,1,128,1,fp8,fp8,0,0.6104640007019043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,4,128,1,float16,fp8,0,0.16586079597473144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,4,128,1,fp8,fp8,0,0.16522560119628907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,8,128,1,float16,float16,0,0.17395360469818116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,8,128,1,float16,fp8,0,0.17316960096359252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,8,128,1,fp8,fp8,0,0.1727887988090515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,40,128,1,float16,float16,0,0.1279647946357727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,40,128,1,float16,fp8,0,0.11829119920730591
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,40,128,1,fp8,fp8,0,0.1178928017616272
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,40,8,128,1,float16,float16,0,0.6554719924926757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,1,128,1,float16,fp8,0,0.08784639835357666
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,1,128,1,fp8,fp8,0,0.08744959831237793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,2,128,1,float16,float16,0,0.0872111976146698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,2,128,1,float16,fp8,0,0.08848159909248351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,2,128,1,fp8,fp8,0,0.08805599808692932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,4,128,1,float16,float16,0,0.08926079869270324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,4,128,1,float16,fp8,0,0.09026399850845337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,4,128,1,fp8,fp8,0,0.08942720293998718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,40,1,128,1,float16,float16,0,0.3067615985870361
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,8,128,1,float16,float16,0,0.09364640116691589
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,8,128,1,float16,fp8,0,0.09325439929962158
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,40,128,1,float16,float16,0,0.07230560183525085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,40,128,1,float16,fp8,0,0.06649600267410279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,40,128,1,fp8,fp8,0,0.06688799858093261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,1,128,1,float16,fp8,0,0.04673120081424713
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,1,128,1,fp8,fp8,0,0.0472815990447998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,2,128,1,float16,float16,0,0.0493151992559433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,2,128,1,float16,fp8,0,0.047844800353050235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,2,128,1,fp8,fp8,0,0.04773919880390167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,4,128,1,float16,float16,0,0.05079839825630188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,4,128,1,float16,fp8,0,0.04833120107650757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,40,128,1,float16,float16,0,0.2417072057723999
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,4,128,1,fp8,fp8,0,0.048414400219917296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,8,128,1,float16,float16,0,0.05313119888305664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,8,128,1,float16,fp8,0,0.050083202123641965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,8,128,1,fp8,fp8,0,0.04936800003051758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,40,128,1,float16,float16,0,0.043201598525047305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,40,128,1,float16,fp8,0,0.037134400010108946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,40,128,1,fp8,fp8,0,0.0380048006772995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,1,128,1,float16,fp8,0,0.029462400078773498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,40,4,128,1,float16,float16,0,0.16612800359725952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,1,128,1,fp8,fp8,0,0.029547199606895447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,2,128,1,float16,float16,0,0.028692799806594848
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,2,128,1,float16,fp8,0,0.02953920066356659
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,2,128,1,fp8,fp8,0,0.029158401489257812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,4,128,1,float16,float16,0,0.029260799288749695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,4,128,1,float16,fp8,0,0.02938399910926819
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,4,128,1,fp8,fp8,0,0.029766398668289184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,8,128,1,float16,float16,0,0.02980319857597351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,8,128,1,float16,fp8,0,0.02994079887866974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,8,128,1,fp8,fp8,0,0.029743999242782593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,40,128,1,float16,fp8,0,0.025854399800300597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,40,128,1,fp8,fp8,0,0.025551998615264894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,1,128,1,float16,float16,0,0.01895360052585602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,1,128,1,float16,fp8,0,0.019940799474716185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,1,128,1,fp8,fp8,0,0.019908800721168518
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,2,128,1,float16,float16,0,0.0189983993768692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,8,128,1,fp8,fp8,0,0.09248639941215515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,2,128,1,fp8,fp8,0,0.020126399397850037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,4,128,1,float16,float16,0,0.019324800372123717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,4,128,1,float16,fp8,0,0.020321600139141083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,4,128,1,fp8,fp8,0,0.020278400182723998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,8,128,1,float16,float16,0,0.019572800397872923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,8,128,1,float16,fp8,0,0.020377600193023683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,8,128,1,fp8,fp8,0,0.020311999320983886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,40,1,128,1,float16,float16,0,0.02852639853954315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,40,1,128,1,float16,fp8,0,1.2051136016845703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,40,1,128,1,float16,float16,0,0.9186176300048828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,40,128,1,float16,float16,0,0.024515199661254882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,40,1,128,1,float16,fp8,0,0.9694399833679199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,40,1,128,1,fp8,fp8,0,0.9756688117980957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,40,2,128,1,float16,float16,0,0.9299599647521972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,40,2,128,1,float16,fp8,0,0.9717823982238769
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,40,2,128,1,float16,fp8,0,0.02022559940814972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,40,1,128,1,float16,float16,0,0.048204800486564635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,40,2,128,1,fp8,fp8,0,0.9726752281188965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,40,4,128,1,float16,float16,0,0.9589792251586914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,40,4,128,1,float16,fp8,0,1.0036895751953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,40,4,128,1,fp8,fp8,0,1.003267192840576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,40,8,128,1,float16,float16,0,1.0230704307556153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,40,128,1,float16,fp8,0,0.6832399845123291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,40,8,128,1,fp8,fp8,0,1.0414239883422851
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,1,128,1,float16,float16,0,0.4667056083679199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,40,128,1,fp8,fp8,0,0.6860864162445068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,1,128,1,float16,fp8,0,0.4918623924255371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,1,128,1,fp8,fp8,0,0.49199838638305665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,2,128,1,float16,float16,0,0.4733232021331787
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,2,128,1,fp8,fp8,0,0.4965616226196289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,4,128,1,float16,float16,0,0.48607678413391114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,4,128,1,float16,fp8,0,0.5059423923492432
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,40,1,128,1,float16,float16,0,0.08671039938926697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,8,128,1,float16,float16,0,0.5217088222503662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,8,128,1,float16,fp8,0,0.5249680042266845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,8,128,1,fp8,fp8,0,0.5251935958862305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,40,128,1,float16,float16,0,0.4039792060852051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,40,128,1,float16,fp8,0,0.3506767988204956
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,1,128,1,float16,float16,0,0.23878240585327148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,1,128,1,float16,fp8,0,0.2539423942565918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,1,128,1,fp8,fp8,0,0.25445120334625243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,2,128,1,float16,fp8,0,0.25444319248199465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,2,128,1,fp8,fp8,0,0.25489919185638427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,4,128,1,float16,float16,0,0.2513808012008667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,4,128,1,float16,fp8,0,0.2614543914794922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,40,8,128,1,float16,fp8,0,1.04202880859375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,4,128,1,fp8,fp8,0,0.26163198947906496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,8,128,1,float16,float16,0,0.2676271915435791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,8,128,1,float16,fp8,0,0.2720016002655029
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,40,128,1,float16,float16,0,0.21043999195098878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,2,128,1,float16,fp8,0,0.49573278427124023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,40,128,1,fp8,fp8,0,0.18286720514297486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,1,128,1,float16,float16,0,0.1264575958251953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,1,128,1,float16,fp8,0,0.13315999507904053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,1,128,1,fp8,fp8,0,0.1332927942276001
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,4,128,1,fp8,fp8,0,0.5096943855285645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,2,128,1,float16,fp8,0,0.13395680189132692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,2,128,1,fp8,fp8,0,0.133787202835083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,4,128,1,float16,float16,0,0.13257759809494019
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,4,128,1,float16,fp8,0,0.13700159788131713
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,4,128,1,fp8,fp8,0,0.13673919439315796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,40,128,1,fp8,fp8,0,0.34872639179229736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,8,128,1,float16,fp8,0,0.14221279621124266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,40,128,1,float16,float16,0,0.1123471975326538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,2,128,1,float16,float16,0,0.2421056032180786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,40,128,1,float16,fp8,0,0.09893599748611451
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,40,128,1,fp8,fp8,0,0.09862719774246216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,1,128,1,float16,float16,0,0.0697376012802124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,1,128,1,float16,fp8,0,0.07235040068626404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,1,128,1,fp8,fp8,0,0.07275519967079162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,2,128,1,float16,float16,0,0.07097280025482178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,2,128,1,float16,fp8,0,0.07329599857330323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,2,128,1,fp8,fp8,0,0.07304000258445739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,4,128,1,float16,float16,0,0.07340319752693177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,4,128,1,float16,fp8,0,0.07495200037956237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,4,128,1,fp8,fp8,0,0.07491040229797363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,40,128,1,float16,fp8,0,0.18343039751052856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,8,128,1,float16,float16,0,0.07750719785690308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,8,128,1,float16,fp8,0,0.07716000080108643
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,40,8,128,1,fp8,fp8,0,0.07726399898529053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,40,128,1,float16,float16,0,0.06374719738960266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,40,128,1,fp8,fp8,0,0.056704002618789676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,2,128,1,float16,float16,0,0.1283184051513672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,1,128,1,float16,float16,0,0.038683199882507326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,1,128,1,float16,fp8,0,0.03999840021133423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,1,128,1,fp8,fp8,0,0.03964639902114868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,2,128,1,float16,float16,0,0.040006399154663086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,2,128,1,float16,fp8,0,0.03988640010356903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,2,128,1,fp8,fp8,0,0.03991200029850006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,4,128,1,float16,float16,0,0.04193919897079468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,4,128,1,float16,fp8,0,0.041126400232315063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,4,128,1,fp8,fp8,0,0.04065600037574768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,8,128,1,float16,fp8,0,0.04259360134601593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,8,128,1,fp8,fp8,0,0.14196959733963013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,40,128,1,float16,float16,0,0.03853920102119446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,40,128,1,float16,fp8,0,0.03150720000267029
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,40,128,1,fp8,fp8,0,0.03141759932041168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,1,128,1,float16,float16,0,0.023588800430297853
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,1,128,1,float16,fp8,0,0.02534399926662445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,1,128,1,fp8,fp8,0,0.025472000241279602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,2,128,1,float16,float16,0,0.023707200586795808
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,2,128,1,float16,fp8,0,0.025252801179885865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,2,128,1,fp8,fp8,0,0.025257599353790284
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,4,128,1,float16,float16,0,0.023849600553512575
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,40,8,128,1,fp8,fp8,0,0.2712496042251587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,4,128,1,float16,fp8,0,0.025491198897361754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,4,128,1,fp8,fp8,0,0.02540160119533539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,8,128,1,float16,fp8,0,0.025766399502754212
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,8,128,1,fp8,fp8,0,0.025863999128341676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,40,128,1,float16,float16,0,0.021910400688648225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,40,128,1,float16,fp8,0,0.05687999725341797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,40,128,1,fp8,fp8,0,0.021976000070571898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,1,128,1,float16,float16,0,0.017182399332523347
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,1,128,1,float16,fp8,0,0.018348799645900728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,1,128,1,fp8,fp8,0,0.018320000171661376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,2,128,1,float16,float16,0,0.017185600101947786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,2,128,1,float16,fp8,0,0.01836480051279068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,40,8,128,1,float16,float16,0,0.14105919599533082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,4,128,1,float16,float16,0,0.01730719953775406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,4,128,1,float16,fp8,0,0.018268799781799315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,4,128,1,fp8,fp8,0,0.018187199532985688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,8,128,1,float16,float16,0,0.017635199427604675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,8,128,1,float16,fp8,0,0.018244799971580506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,8,128,1,fp8,fp8,0,0.01839359998703003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,40,128,1,float16,float16,0,0.01604959964752197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,40,128,1,fp8,fp8,0,0.016358399391174318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,1,128,1,float16,float16,0,0.014528000354766845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,1,128,1,float16,fp8,0,0.015212799608707427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,1,128,1,fp8,fp8,0,0.015316799283027649
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,40,40,128,1,float16,float16,0,0.7894320011138916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,2,128,1,float16,float16,0,0.014542399346828461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,2,128,1,fp8,fp8,0,0.015148800611495972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,2,128,1,float16,fp8,0,0.015131199359893798
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,4,128,1,float16,float16,0,0.014630399644374847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,4,128,1,float16,fp8,0,0.015086400508880615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,8,128,1,float16,float16,0,0.014766399562358857
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,8,128,1,float16,fp8,0,0.01568000018596649
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,8,128,1,fp8,fp8,0,0.015278400480747223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,40,128,1,float16,fp8,0,0.022078399360179902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,40,1,128,1,float16,fp8,0,0.4741231918334961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,8,128,1,float16,float16,0,0.04456959962844849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,40,1,128,1,fp8,fp8,0,0.4743663787841797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,40,2,128,1,float16,float16,0,0.442091178894043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,40,128,1,float16,fp8,0,0.01635199934244156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,40,2,128,1,float16,fp8,0,0.4773359775543213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,40,2,128,1,fp8,fp8,0,0.4777215957641602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,40,4,128,1,float16,float16,0,0.4563039779663086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,40,4,128,1,float16,fp8,0,0.48909759521484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,40,4,128,1,fp8,fp8,0,0.015166400372982025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,40,4,128,1,fp8,fp8,0,0.48926877975463867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,40,8,128,1,float16,float16,0,0.024689599871635437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,40,8,128,1,float16,fp8,0,0.5148064136505127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,40,8,128,1,fp8,fp8,0,0.5114016056060791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,40,128,1,float16,float16,0,0.3835664033889771
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,40,128,1,float16,fp8,0,0.34589760303497313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,40,1,128,1,float16,float16,0,0.4356192111968994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,40,128,1,fp8,fp8,0,0.34729440212249757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,1,128,1,float16,fp8,0,0.24141280651092528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,1,128,1,fp8,fp8,0,0.24036641120910646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,2,128,1,float16,fp8,0,0.24273920059204102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,2,128,1,fp8,fp8,0,0.24234719276428224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,40,2,128,1,fp8,fp8,0,0.018376000225543976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,4,128,1,float16,fp8,0,0.24874238967895507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,4,128,1,fp8,fp8,0,0.24779999256134033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,40,8,128,1,fp8,fp8,0,0.042387199401855466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,8,128,1,float16,float16,0,0.24753279685974122
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,8,128,1,float16,fp8,0,0.2618256092071533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,8,128,1,fp8,fp8,0,0.2627055883407593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,40,128,1,float16,float16,0,0.20214080810546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,40,128,1,fp8,fp8,0,0.18052639961242675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,1,128,1,float16,float16,0,0.11779040098190308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,1,128,1,float16,fp8,0,0.1280735969543457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,2,128,1,float16,float16,0,0.12001760005950927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,2,128,1,float16,fp8,0,0.1291599988937378
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,2,128,1,fp8,fp8,0,0.12942880392074585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,40,8,128,1,float16,float16,0,0.4863296031951904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,4,128,1,float16,float16,0,0.12505760192871093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,4,128,1,float16,fp8,0,0.1315791964530945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,4,128,1,fp8,fp8,0,0.1313423991203308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,8,128,1,float16,float16,0,0.13267680406570434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,8,128,1,float16,fp8,0,0.13816800117492675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,8,128,1,fp8,fp8,0,0.1383280038833618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,1,128,1,float16,float16,0,0.222542405128479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,40,128,1,float16,float16,0,0.1070863962173462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,2,128,1,float16,float16,0,0.2265455961227417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,40,128,1,fp8,fp8,0,0.0952239990234375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,1,128,1,float16,float16,0,0.066075199842453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,1,128,1,float16,fp8,0,0.06876159906387329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,1,128,1,fp8,fp8,0,0.06848639845848084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,40,4,128,1,float16,float16,0,0.23322079181671143
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,2,128,1,fp8,fp8,0,0.0691424012184143
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,4,128,1,float16,float16,0,0.06910399794578552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,4,128,1,float16,fp8,0,0.07042719721794129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,4,128,1,fp8,fp8,0,0.07043520212173462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,8,128,1,float16,float16,0,0.07288640141487121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,8,128,1,fp8,fp8,0,0.07257440090179443
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,40,128,1,float16,fp8,0,0.18059840202331542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,40,128,1,float16,float16,0,0.060022401809692386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,40,128,1,float16,fp8,0,0.05113760232925415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,40,128,1,fp8,fp8,0,0.050804799795150755
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,1,128,1,float16,float16,0,0.03479839861392975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,1,128,1,float16,fp8,0,0.034755200147628784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,40,1,128,1,fp8,fp8,0,0.12873760461807252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,1,128,1,fp8,fp8,0,0.03531840145587921
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,2,128,1,float16,float16,0,0.036632001399993896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,2,128,1,float16,fp8,0,0.03517119884490967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,2,128,1,fp8,fp8,0,0.03538079857826233
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,4,128,1,float16,float16,0,0.03868640065193176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,4,128,1,fp8,fp8,0,0.035718399286270144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,8,128,1,float16,float16,0,0.04156799912452698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,8,128,1,float16,fp8,0,0.03702400028705597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,8,128,1,fp8,fp8,0,0.036817601323127745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,40,128,1,float16,float16,0,0.036471998691558837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,40,128,1,float16,fp8,0,0.02826719880104065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,40,128,1,fp8,fp8,0,0.02853600084781647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,1,128,1,float16,float16,0,0.021819199621677398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,1,128,1,float16,fp8,0,0.02391680032014847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,1,128,1,fp8,fp8,0,0.023478400707244874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,2,128,1,float16,fp8,0,0.023497599363327026
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,2,128,1,fp8,fp8,0,0.023505599796772005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,4,128,1,float16,float16,0,0.022555199265480042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,4,128,1,float16,fp8,0,0.023827199637889863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,4,128,1,fp8,fp8,0,0.0238319993019104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,8,128,1,float16,float16,0,0.02346560060977936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,8,128,1,float16,fp8,0,0.02412160038948059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,8,128,1,fp8,fp8,0,0.02399519979953766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,2,128,1,float16,float16,0,0.06712639927864075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,2,128,1,float16,fp8,0,0.06957119703292847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,40,128,1,float16,fp8,0,0.019702400267124175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,40,128,1,fp8,fp8,0,0.01974399983882904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,1,128,1,float16,float16,0,0.01720159947872162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,1,128,1,float16,fp8,0,0.018427200615406036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,1,128,1,fp8,fp8,0,0.01833920031785965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,2,128,1,float16,float16,0,0.017420800030231477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,2,128,1,float16,fp8,0,0.01842239946126938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,8,128,1,float16,fp8,0,0.07290080189704895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,4,128,1,float16,float16,0,0.01761920005083084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,4,128,1,float16,fp8,0,0.01851679980754852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,4,128,1,fp8,fp8,0,0.018353599309921264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,8,128,1,float16,fp8,0,0.018561600148677825
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,8,128,1,fp8,fp8,0,0.018593600392341612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,40,128,1,float16,float16,0,0.01446560025215149
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,40,128,1,float16,fp8,0,0.014585599303245544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,40,128,1,fp8,fp8,0,0.014430400729179383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,1,128,1,float16,float16,0,0.012731200456619263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,1,128,1,float16,fp8,0,0.013299199938774108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,1,128,1,fp8,fp8,0,0.01348160058259964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,2,128,1,float16,fp8,0,0.013383999466896057
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,2,128,1,fp8,fp8,0,0.013247999548912048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,40,4,128,1,float16,fp8,0,0.035815998911857605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,4,128,1,float16,float16,0,0.01279519945383072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,4,128,1,float16,fp8,0,0.013350400328636169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,4,128,1,fp8,fp8,0,0.013169600069522858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,8,128,1,float16,float16,0,0.012904000282287598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,8,128,1,float16,fp8,0,0.013348799943923951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,8,128,1,fp8,fp8,0,0.013465599715709686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,40,128,1,float16,float16,0,0.013595199584960938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,40,128,1,float16,fp8,0,0.014051200449466705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,40,128,1,fp8,fp8,0,0.01406719982624054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,1,128,1,float16,float16,0,0.012505599856376648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,1,128,1,float16,fp8,0,0.013291199505329133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,1,128,1,fp8,fp8,0,0.013132800161838532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,2,128,1,float16,float16,0,0.012588800489902496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,2,128,1,float16,fp8,0,0.013174399733543396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,2,128,1,fp8,fp8,0,0.013184000551700593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,4,128,1,float16,float16,0,0.012583999335765839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,4,128,1,float16,fp8,0,0.01313440054655075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,4,128,1,fp8,fp8,0,0.012982399761676788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,8,128,1,float16,float16,0,0.012780800461769104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,40,40,128,1,float16,fp8,0,0.09567679762840271
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,8,128,1,fp8,fp8,0,0.012919999659061432
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,40,128,1,float16,float16,0,0.020707200467586517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,40,1,128,1,float16,float16,0,0.22344319820404052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,40,1,128,1,float16,fp8,0,0.23840000629425048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,2,128,1,fp8,fp8,0,0.018427200615406036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,40,8,128,1,float16,float16,0,0.017985600233078002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,40,2,128,1,float16,float16,0,0.22371840476989746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,40,2,128,1,float16,fp8,0,0.24107201099395753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,40,2,128,1,fp8,fp8,0,0.242792010307312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,40,4,128,1,float16,float16,0,0.2310336112976074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,40,4,128,1,float16,fp8,0,0.24657440185546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,40,4,128,1,fp8,fp8,0,0.24782719612121581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,40,8,128,1,float16,float16,0,0.27853438854217527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,40,8,128,1,float16,fp8,0,0.2907520055770874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,40,8,128,1,fp8,fp8,0,0.29202721118927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,40,128,1,float16,float16,0,0.2720864057540894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,40,128,1,float16,fp8,0,0.2626944065093994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,1,128,1,float16,float16,0,0.11738400459289551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,1,128,1,float16,fp8,0,0.12576160430908204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,40,128,1,fp8,fp8,0,0.2636847972869873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,1,128,1,fp8,fp8,0,0.1262096047401428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,2,128,1,float16,float16,0,0.117958402633667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,2,128,1,float16,fp8,0,0.12936320304870605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,2,128,1,fp8,fp8,0,0.1271008014678955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,4,128,1,float16,float16,0,0.12264959812164307
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,40,8,128,1,float16,fp8,0,0.01305440068244934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,4,128,1,float16,fp8,0,0.13199520111083984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,4,128,1,fp8,fp8,0,0.12977440357208253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,8,128,1,float16,float16,0,0.14536960124969484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,8,128,1,float16,fp8,0,0.1535856008529663
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,40,128,1,float16,float16,0,0.14315359592437743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,40,128,1,float16,fp8,0,0.13937439918518066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,40,128,1,fp8,fp8,0,0.13941600322723388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,1,128,1,float16,fp8,0,0.06934400200843811
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,1,128,1,fp8,fp8,0,0.0691536009311676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,2,128,1,float16,float16,0,0.06832960247993469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,2,128,1,float16,fp8,0,0.07016800045967102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,2,128,1,fp8,fp8,0,0.06988000273704528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,4,128,1,float16,float16,0,0.0693552017211914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,4,128,1,float16,fp8,0,0.07134240269660949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,4,128,1,fp8,fp8,0,0.07160639762878418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,8,128,1,float16,float16,0,0.08191519975662231
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,8,128,1,float16,fp8,0,0.08164160251617432
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,8,128,1,fp8,fp8,0,0.08203999996185303
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,40,128,1,float16,float16,0,0.07935839891433716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,40,128,1,float16,fp8,0,0.07424319982528686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,1,128,1,float16,fp8,0,0.03531199991703034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,1,128,1,fp8,fp8,0,0.035436800122261046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,2,128,1,float16,float16,0,0.036559998989105225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,2,128,1,float16,fp8,0,0.03527680039405823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,2,128,1,fp8,fp8,0,0.03523840010166168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,4,128,1,float16,float16,0,0.03883999884128571
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,4,128,1,float16,fp8,0,0.03572640120983124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,40,2,128,1,float16,float16,0,0.022492800652980805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,4,128,1,fp8,fp8,0,0.03563840091228485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,8,128,1,float16,float16,0,0.04535680115222931
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,8,128,1,float16,fp8,0,0.041910400986671446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,8,128,1,fp8,fp8,0,0.04184800088405609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,40,128,1,float16,float16,0,0.04396960139274597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,40,128,1,float16,fp8,0,0.03796960115432739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,40,128,1,fp8,fp8,0,0.03732640147209167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,1,128,1,float16,float16,0,0.021953600645065307
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,1,128,1,float16,fp8,0,0.023742400109767914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,1,128,1,fp8,fp8,0,0.023708799481391908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,2,128,1,float16,float16,0,0.022064000368118286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,40,8,128,1,fp8,fp8,0,0.15366560220718384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,2,128,1,fp8,fp8,0,0.02377920001745224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,4,128,1,float16,float16,0,0.022599999606609345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,4,128,1,float16,fp8,0,0.023763200640678404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,4,128,1,fp8,fp8,0,0.023830400407314302
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,8,128,1,float16,float16,0,0.02253440022468567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,8,128,1,float16,fp8,0,0.02412160038948059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,8,128,1,fp8,fp8,0,0.023824000358581544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,40,128,1,float16,float16,0,0.02353599965572357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,40,128,1,float16,fp8,0,0.023955200612545014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,40,128,1,fp8,fp8,0,0.024102400243282317
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,1,128,1,float16,float16,0,0.017080000042915343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,40,1,128,1,fp8,fp8,0,0.239900803565979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,1,128,1,fp8,fp8,0,0.018475200235843658
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,2,128,1,float16,float16,0,0.017105600237846373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,2,128,1,fp8,fp8,0,0.018320000171661376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,4,128,1,float16,float16,0,0.017395199835300447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,4,128,1,float16,fp8,0,0.01852799952030182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,4,128,1,fp8,fp8,0,0.018507200479507446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,40,2,128,1,float16,float16,0,0.012664000689983367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,8,128,1,float16,float16,0,0.017318400740623473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,8,128,1,fp8,fp8,0,0.018356800079345703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,40,128,1,float16,float16,0,0.017582400143146514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,40,128,1,float16,fp8,0,0.018571199476718904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,40,128,1,fp8,fp8,0,0.018572799861431122
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,1,128,1,float16,float16,0,0.012689599394798278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,1,128,1,float16,fp8,0,0.013291199505329133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,1,128,1,fp8,fp8,0,0.01316000074148178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,2,128,1,float16,float16,0,0.012495999783277511
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,2,128,1,float16,fp8,0,0.013099199533462525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,2,128,1,fp8,fp8,0,0.013089600205421447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,4,128,1,float16,float16,0,0.012585599720478059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,4,128,1,float16,fp8,0,0.0133200004696846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,4,128,1,fp8,fp8,0,0.013307200372219085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,40,2,128,1,float16,fp8,0,0.023849600553512575
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,8,128,1,float16,float16,0,0.012889599800109864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,40,1,128,1,float16,float16,0,0.06614559888839722
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,8,128,1,fp8,fp8,0,0.013483199477195739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,40,128,1,float16,float16,0,0.013225600123405457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,40,128,1,float16,fp8,0,0.013816000521183014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,40,128,1,fp8,fp8,0,0.013944000005722046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,1,128,1,float16,float16,0,0.012452799826860428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,2,128,1,float16,fp8,0,0.018580800294876097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,1,128,1,fp8,fp8,0,0.013172799348831176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,2,128,1,float16,float16,0,0.01250240057706833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,40,128,1,fp8,fp8,0,0.07433279752731323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,2,128,1,float16,fp8,0,0.013124799728393555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,2,128,1,fp8,fp8,0,0.01319040060043335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,4,128,1,float16,float16,0,0.012494400143623352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,4,128,1,float16,fp8,0,0.01300320029258728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,4,128,1,fp8,fp8,0,0.01295360028743744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,8,128,1,float16,fp8,0,0.018593600392341612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,8,128,1,float16,float16,0,0.012374400347471236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,8,128,1,float16,fp8,0,0.013020800054073333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,8,128,1,fp8,fp8,0,0.01308799982070923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,40,128,1,float16,fp8,0,0.013692800700664521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,40,128,1,fp8,fp8,0,0.013748799264431
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,1,128,1,float16,float16,0,0.012337599694728852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,1,128,1,float16,fp8,0,0.012852799892425538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,1,128,1,fp8,fp8,0,0.012964800000190735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,2,128,1,float16,float16,0,0.012439999729394913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,2,128,1,float16,fp8,0,0.013012799620628356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,2,128,1,fp8,fp8,0,0.013100799918174744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,4,128,1,float16,float16,0,0.012476799637079239
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,4,128,1,float16,fp8,0,0.012883199751377106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,4,128,1,fp8,fp8,0,0.013107199966907502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,8,128,1,float16,float16,0,0.012350399792194367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,8,128,1,float16,fp8,0,0.013067199289798737
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,8,128,1,fp8,fp8,0,0.013128000497817992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,40,1,128,1,float16,float16,0,0.11815199851989747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,40,1,128,1,float16,fp8,0,0.1293488025665283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,40,1,128,1,fp8,fp8,0,0.1292080044746399
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,40,2,128,1,float16,float16,0,0.12136000394821167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,40,2,128,1,float16,fp8,0,0.13018239736557008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,40,2,128,1,fp8,fp8,0,0.13000320196151732
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,40,8,128,1,float16,fp8,0,0.01358720064163208
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,40,4,128,1,float16,fp8,0,0.1492319941520691
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,40,4,128,1,fp8,fp8,0,0.1489408016204834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,40,8,128,1,float16,float16,0,0.17106879949569703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,40,1,128,1,float16,fp8,0,0.01839359998703003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,40,8,128,1,fp8,fp8,0,0.1847007989883423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,40,1,128,1,float16,fp8,0,0.013177600502967835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,40,128,1,float16,float16,0,0.21860959529876708
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,40,128,1,float16,fp8,0,0.22956960201263427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,1,128,1,float16,float16,0,0.06674079895019532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,40,128,1,fp8,fp8,0,0.22854080200195312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,1,128,1,float16,fp8,0,0.07012159824371338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,1,128,1,fp8,fp8,0,0.07023199796676635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,2,128,1,float16,float16,0,0.0676800012588501
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,2,128,1,float16,fp8,0,0.07085440158843995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,2,128,1,fp8,fp8,0,0.07084959745407104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,4,128,1,float16,float16,0,0.07586719989776611
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,4,128,1,float16,fp8,0,0.08008959889411926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,4,128,1,fp8,fp8,0,0.08001279830932617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,8,128,1,float16,float16,0,0.0926639974117279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,8,128,1,float16,fp8,0,0.09949600100517272
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,40,8,128,1,fp8,fp8,0,0.0993552029132843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,40,128,1,float16,float16,0,0.11702239513397217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,40,128,1,float16,fp8,0,0.12144479751586915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,40,128,1,fp8,fp8,0,0.12070080041885375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,1,128,1,float16,fp8,0,0.035622400045394895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,1,128,1,fp8,fp8,0,0.03555839955806732
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,2,128,1,float16,float16,0,0.03650240004062653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,2,128,1,fp8,fp8,0,0.035764798521995544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,4,128,1,float16,float16,0,0.04312160015106201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,4,128,1,float16,fp8,0,0.04043680131435394
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,4,128,1,fp8,fp8,0,0.04057280123233795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,8,128,1,float16,float16,0,0.05289599895477295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,8,128,1,float16,fp8,0,0.0515936017036438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,8,128,1,fp8,fp8,0,0.05117279887199402
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,40,128,1,float16,float16,0,0.06313120126724243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,40,128,1,float16,fp8,0,0.06117119789123535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,40,128,1,fp8,fp8,0,0.06028320193290711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,40,4,128,1,float16,float16,0,0.1392240047454834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,1,128,1,float16,fp8,0,0.023878400027751923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,1,128,1,fp8,fp8,0,0.0239439994096756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,2,128,1,float16,float16,0,0.022417600452899932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,2,128,1,float16,fp8,0,0.0239424005150795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,2,128,1,fp8,fp8,0,0.024091200530529024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,4,128,1,float16,float16,0,0.02221119999885559
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,4,128,1,float16,fp8,0,0.02423200011253357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,40,8,128,1,float16,fp8,0,0.18470879793167114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,8,128,1,float16,float16,0,0.02672959864139557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,8,128,1,float16,fp8,0,0.02906239926815033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,8,128,1,fp8,fp8,0,0.02889440059661865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,40,128,1,float16,float16,0,0.031108799576759338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,40,128,1,float16,fp8,0,0.03287999927997589
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,40,128,1,fp8,fp8,0,0.03312639892101288
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,1,128,1,float16,float16,0,0.0169855996966362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,40,1,128,1,float16,float16,0,0.03447839915752411
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,1,128,1,fp8,fp8,0,0.018160000443458557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,2,128,1,float16,float16,0,0.01719360053539276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,2,128,1,float16,fp8,0,0.018193599581718446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,2,128,1,fp8,fp8,0,0.0182559996843338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,40,40,128,1,float16,float16,0,0.013035200536251068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,4,128,1,float16,fp8,0,0.018483200669288637
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,4,128,1,fp8,fp8,0,0.018488000333309173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,8,128,1,float16,float16,0,0.017497600615024568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,1,128,1,float16,float16,0,0.034750398993492124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,8,128,1,float16,fp8,0,0.018662400543689728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,8,128,1,fp8,fp8,0,0.018854400515556334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,40,128,1,float16,float16,0,0.021564799547195434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,40,128,1,float16,fp8,0,0.023240000009536743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,40,2,128,1,float16,fp8,0,0.035416001081466676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,1,128,1,float16,float16,0,0.01273760050535202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,1,128,1,float16,fp8,0,0.013516800105571746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,1,128,1,fp8,fp8,0,0.01327199935913086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,2,128,1,float16,float16,0,0.012902399897575379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,2,128,1,float16,fp8,0,0.013433599472045898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,2,128,1,fp8,fp8,0,0.01371839940547943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,4,128,1,float16,fp8,0,0.01356000006198883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,4,128,1,fp8,fp8,0,0.013651199638843536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,8,128,1,float16,float16,0,0.013191999495029449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,8,128,1,float16,fp8,0,0.013689599931240082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,1,128,1,float16,float16,0,0.02232320010662079
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,8,128,1,fp8,fp8,0,0.013784000277519226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,40,128,1,float16,float16,0,0.017140799760818483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,40,128,1,float16,fp8,0,0.01812160015106201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,40,128,1,fp8,fp8,0,0.018326400220394133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,1,128,1,float16,fp8,0,0.01311199963092804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,40,4,128,1,fp8,fp8,0,0.02415039986371994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,1,128,1,fp8,fp8,0,0.013129599392414093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,2,128,1,float16,float16,0,0.012489599734544754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,2,128,1,float16,fp8,0,0.01313440054655075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,2,128,1,fp8,fp8,0,0.01308639943599701
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,4,128,1,float16,float16,0,0.012460800260305405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,4,128,1,float16,fp8,0,0.012988799810409546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,8,128,1,float16,float16,0,0.012415999919176102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,8,128,1,float16,fp8,0,0.01316159963607788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,8,128,1,fp8,fp8,0,0.013329599797725678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,4,128,1,float16,float16,0,0.017105600237846373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,40,128,1,float16,float16,0,0.013124799728393555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,40,128,1,float16,fp8,0,0.013801600039005279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,40,128,1,fp8,fp8,0,0.01398719996213913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,1,128,1,float16,float16,0,0.012438400089740754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,1,128,1,float16,fp8,0,0.013079999387264252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,1,128,1,fp8,fp8,0,0.013022400438785553
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,2,128,1,float16,fp8,0,0.01297920048236847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,2,128,1,fp8,fp8,0,0.012857599556446076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,4,128,1,float16,float16,0,0.012100800126791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,4,128,1,float16,fp8,0,0.012831999361515046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,4,128,1,fp8,fp8,0,0.01276479959487915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,8,128,1,float16,float16,0,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,8,128,1,float16,fp8,0,0.012955200672149659
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,8,128,1,fp8,fp8,0,0.01302880048751831
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,40,128,1,float16,float16,0,0.012943999469280243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,4,128,1,float16,float16,0,0.012721599638462066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,40,128,1,float16,fp8,0,0.013524800539016724
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,1,128,1,float16,float16,0,0.012008000165224075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,1,128,1,float16,fp8,0,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,1,128,1,fp8,fp8,0,0.012774400413036346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,2,128,1,float16,float16,0,0.012151999771595002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,2,128,1,float16,fp8,0,0.012774400413036346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,2,128,1,fp8,fp8,0,0.01279360055923462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,4,128,1,float16,float16,0,0.01210559979081154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,4,128,1,float16,fp8,0,0.012670400738716125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,4,128,1,fp8,fp8,0,0.012593600153923034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,8,128,1,float16,float16,0,0.011947199702262878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,8,128,1,float16,fp8,0,0.012811200320720672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,8,128,1,fp8,fp8,0,0.012814399600028992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,4,128,1,fp8,fp8,0,0.013196800649166108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,40,1,128,1,float16,float16,0,0.06874719858169556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,40,1,128,1,float16,fp8,0,0.018303999304771425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,40,1,128,1,float16,fp8,0,0.07132800221443177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,40,1,128,1,fp8,fp8,0,0.0714143991470337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,40,2,128,1,float16,float16,0,0.07569760084152222
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,40,2,128,1,float16,fp8,0,0.07993280291557311
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,40,4,128,1,float16,float16,0,0.0898256003856659
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,40,4,128,1,float16,fp8,0,0.09842240214347839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,40,4,128,1,fp8,fp8,0,0.09844319820404053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,40,2,128,1,float16,float16,0,0.012299200147390365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,40,8,128,1,float16,float16,0,0.09572960138320923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,40,8,128,1,float16,fp8,0,0.10240479707717895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,40,8,128,1,fp8,fp8,0,0.10255839824676513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,40,128,1,float16,float16,0,0.1929919958114624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,1,128,1,float16,float16,0,0.03600000143051148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,1,128,1,float16,fp8,0,0.036447998881340024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,1,128,1,fp8,fp8,0,0.03653280138969421
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,40,40,128,1,fp8,fp8,0,0.023075200617313385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,2,128,1,float16,float16,0,0.04094240069389343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,2,128,1,float16,fp8,0,0.04058400094509125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,2,128,1,fp8,fp8,0,0.04051199853420258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,4,128,1,float16,float16,0,0.05097759962081909
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,4,128,1,float16,fp8,0,0.050241601467132566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,4,128,1,fp8,fp8,0,0.05099200010299683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,8,128,1,float16,float16,0,0.05479199886322021
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,8,128,1,fp8,fp8,0,0.05222079753875732
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,40,40,128,1,fp8,fp8,0,0.013654400408267976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,40,128,1,float16,fp8,0,0.10941280126571655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,40,128,1,fp8,fp8,0,0.10655360221862793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,1,128,1,float16,float16,0,0.02276480048894882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,1,128,1,float16,fp8,0,0.02446240037679672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,1,128,1,fp8,fp8,0,0.024512000381946564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,2,128,1,float16,float16,0,0.02274720072746277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,2,128,1,float16,fp8,0,0.024702399969100952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,40,1,128,1,float16,float16,0,0.012278400361537933
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,2,128,1,fp8,fp8,0,0.024539199471473695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,4,128,1,float16,float16,0,0.02669120132923126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,4,128,1,float16,fp8,0,0.029070401191711427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,4,128,1,fp8,fp8,0,0.02914080023765564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,8,128,1,float16,float16,0,0.026638400554656983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,8,128,1,float16,fp8,0,0.02914400100708008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,8,128,1,fp8,fp8,0,0.029083201289176942
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,40,128,1,float16,float16,0,0.05028480291366577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,40,128,1,float16,fp8,0,0.05639680027961731
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,40,128,1,fp8,fp8,0,0.05644000172615051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,1,128,1,float16,float16,0,0.0176704004406929
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,1,128,1,float16,fp8,0,0.018863999843597413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,1,128,1,fp8,fp8,0,0.01881439983844757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,2,128,1,float16,float16,0,0.017548799514770508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,2,128,1,float16,fp8,0,0.01880960017442703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,2,128,1,fp8,fp8,0,0.018859200179576874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,4,128,1,float16,float16,0,0.017825600504875184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,4,128,1,fp8,fp8,0,0.019113600254058838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,8,128,1,float16,float16,0,0.017793600261211396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,8,128,1,float16,fp8,0,0.018667200207710268
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,8,128,1,fp8,fp8,0,0.0186271995306015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,40,128,1,float16,float16,0,0.02930240035057068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,40,128,1,float16,fp8,0,0.032488000392913816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,40,128,1,fp8,fp8,0,0.21269600391387938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,40,128,1,fp8,fp8,0,0.03259040117263794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,1,128,1,float16,float16,0,0.01297599971294403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,1,128,1,float16,fp8,0,0.01374399960041046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,1,128,1,fp8,fp8,0,0.01353919953107834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,2,128,1,float16,float16,0,0.01313599944114685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,2,128,1,float16,fp8,0,0.013580800592899322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,2,128,1,fp8,fp8,0,0.013606399297714233
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,4,128,1,float16,float16,0,0.012993599474430084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,4,128,1,float16,fp8,0,0.013635200262069703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,4,128,1,fp8,fp8,0,0.013651199638843536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,8,128,1,float16,float16,0,0.013097600638866424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,8,128,1,float16,fp8,0,0.01363999992609024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,40,8,128,1,fp8,fp8,0,0.013459199666976928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,40,128,1,float16,float16,0,0.020983999967575072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,40,128,1,float16,fp8,0,0.02279040068387985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,40,128,1,fp8,fp8,0,0.022894400358200073
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,1,128,1,float16,float16,0,0.012404800206422806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,1,128,1,float16,fp8,0,0.013150399923324585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,1,128,1,fp8,fp8,0,0.013105599582195282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,2,128,1,float16,float16,0,0.01239679977297783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,2,128,1,float16,fp8,0,0.013033600151538849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,2,128,1,fp8,fp8,0,0.013203200697898865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,4,128,1,float16,float16,0,0.012452799826860428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,4,128,1,float16,fp8,0,0.013171200454235078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,4,128,1,fp8,fp8,0,0.013204799592494964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,8,128,1,float16,float16,0,0.012252800166606903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,8,128,1,float16,fp8,0,0.013031999766826629
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,40,8,128,1,fp8,fp8,0,0.013142399489879608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,40,128,1,float16,float16,0,0.01660960018634796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,40,128,1,float16,fp8,0,0.018104000389575957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,40,128,1,fp8,fp8,0,0.01791680008172989
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,1,128,1,float16,float16,0,0.012296000123023986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,1,128,1,float16,fp8,0,0.012918399274349212
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,1,128,1,fp8,fp8,0,0.012715199589729309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,2,128,1,float16,float16,0,0.012342400103807449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,2,128,1,fp8,fp8,0,0.013067199289798737
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,4,128,1,float16,float16,0,0.012185599654912949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,4,128,1,float16,fp8,0,0.013079999387264252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,4,128,1,fp8,fp8,0,0.013079999387264252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,8,128,1,float16,float16,0,0.011823999881744384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,8,128,1,float16,fp8,0,0.012740799784660339
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,8,128,1,fp8,fp8,0,0.012768000364303589
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,40,128,1,float16,float16,0,0.01268800050020218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,40,128,1,float16,fp8,0,0.21398239135742186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,40,128,1,fp8,fp8,0,0.013896000385284425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,1,128,1,float16,float16,0,0.012064000219106674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,1,128,1,float16,fp8,0,0.012689599394798278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,1,128,1,fp8,fp8,0,0.012681600451469422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,40,8,128,1,float16,fp8,0,0.052267199754714964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,2,128,1,float16,float16,0,0.011947199702262878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,2,128,1,float16,fp8,0,0.012907199561595917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,2,128,1,fp8,fp8,0,0.01265760064125061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,4,128,1,float16,float16,0,0.012166400253772736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,4,128,1,fp8,fp8,0,0.01271200031042099
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,8,128,1,float16,float16,0,0.011947199702262878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,8,128,1,float16,fp8,0,0.012625600397586822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,8,128,1,fp8,fp8,0,0.012857599556446076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,40,128,1,float16,float16,0,0.012928000092506409
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,40,128,1,fp8,fp8,0,0.013779200613498688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,1,128,1,float16,float16,0,0.011846400052309036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,1,128,1,float16,fp8,0,0.012608000636100769
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,1,128,1,fp8,fp8,0,0.012647999823093415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,2,128,1,float16,float16,0,0.01202239990234375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,2,128,1,float16,fp8,0,0.01268479973077774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,2,128,1,fp8,fp8,0,0.012702399492263794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,4,128,1,float16,float16,0,0.011990399658679962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,4,128,1,float16,fp8,0,0.012667199969291687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,4,128,1,fp8,fp8,0,0.012588800489902496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,40,2,128,1,float16,fp8,0,0.012963199615478515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,8,128,1,float16,float16,0,0.011699199676513672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,8,128,1,float16,fp8,0,0.012535999715328216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,8,128,1,fp8,fp8,0,0.012432000041007996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,40,2,128,1,fp8,fp8,0,0.08010240197181702
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,40,1,128,1,float16,float16,0,0.02141599953174591
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,40,1,128,1,float16,fp8,0,0.02289759963750839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,40,1,128,1,fp8,fp8,0,0.022540800273418427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,40,2,128,1,float16,float16,0,0.028295999765396117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,40,2,128,1,float16,fp8,0,0.031384000182151796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,40,2,128,1,fp8,fp8,0,0.03177599906921387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,40,4,128,1,float16,float16,0,0.043079999089241025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,40,4,128,1,float16,fp8,0,0.04927999973297119
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,40,4,128,1,fp8,fp8,0,0.04885919988155365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,40,8,128,1,float16,float16,0,0.07167680263519287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,40,8,128,1,float16,fp8,0,0.08435360193252564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,40,128,1,float16,fp8,0,0.013484799861907959
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,40,128,1,float16,float16,0,0.155459201335907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,40,128,1,float16,fp8,0,0.18920160531997682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,40,128,1,fp8,fp8,0,0.18881599903106688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,1,128,1,float16,fp8,0,0.0159183993935585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,1,128,1,fp8,fp8,0,0.015795199573040007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,40,40,128,1,float16,float16,0,0.10263359546661377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,40,4,128,1,float16,fp8,0,0.012908799946308136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,2,128,1,float16,fp8,0,0.02011999934911728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,2,128,1,fp8,fp8,0,0.02028159946203232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,40,40,128,1,float16,fp8,0,0.013435199856758118
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,4,128,1,float16,fp8,0,0.029201599955558776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,4,128,1,fp8,fp8,0,0.02906399965286255
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,8,128,1,float16,float16,0,0.04063520133495331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,8,128,1,float16,fp8,0,0.04688799977302551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,40,128,1,float16,float16,0,0.08486080169677734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,40,128,1,float16,fp8,0,0.10182720422744751
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,40,128,1,fp8,fp8,0,0.10168479681015015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,1,128,1,float16,float16,0,0.01316480040550232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,1,128,1,float16,fp8,0,0.0142752006649971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,1,128,1,fp8,fp8,0,0.014126400649547576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,40,4,128,1,float16,fp8,0,0.019044800102710722
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,2,128,1,float16,float16,0,0.013651199638843536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,2,128,1,float16,fp8,0,0.01414559930562973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,4,128,1,float16,float16,0,0.017263999581336974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,4,128,1,float16,fp8,0,0.0186831995844841
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,4,128,1,fp8,fp8,0,0.01866399943828583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,8,128,1,float16,float16,0,0.024478399753570558
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,40,8,128,1,fp8,fp8,0,0.0849407970905304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,8,128,1,float16,fp8,0,0.027430400252342224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,8,128,1,fp8,fp8,0,0.027540799975395203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,40,128,1,float16,fp8,0,0.055307197570800784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,1,128,1,float16,float16,0,0.014788800477981567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,1,128,1,float16,float16,0,0.012430399656295776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,1,128,1,float16,fp8,0,0.013228799402713775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,2,128,1,float16,float16,0,0.012681600451469422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,2,128,1,float16,fp8,0,0.013462400436401368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,2,128,1,fp8,fp8,0,0.013368000090122224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,4,128,1,float16,float16,0,0.0125231996178627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,4,128,1,float16,float16,0,0.025937598943710328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,4,128,1,float16,fp8,0,0.013417600095272065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,8,128,1,float16,float16,0,0.016451199352741242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,8,128,1,float16,fp8,0,0.017875200510025023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,8,128,1,fp8,fp8,0,0.017897599935531618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,8,128,1,fp8,fp8,0,0.04660800099372864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,40,128,1,float16,float16,0,0.02837119996547699
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,40,128,1,float16,fp8,0,0.03164480030536652
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,40,128,1,fp8,fp8,0,0.03194560110569
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,1,128,1,float16,float16,0,0.011673600226640702
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,1,128,1,float16,fp8,0,0.012427199631929398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,1,128,1,fp8,fp8,0,0.01244800016283989
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,2,128,1,float16,float16,0,0.012008000165224075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,2,128,1,float16,fp8,0,0.012771199643611907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,2,128,1,fp8,fp8,0,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,4,128,1,float16,float16,0,0.011999999731779098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,4,128,1,float16,fp8,0,0.01289760023355484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,4,128,1,fp8,fp8,0,0.012852799892425538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,8,128,1,float16,float16,0,0.012415999919176102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,8,128,1,float16,fp8,0,0.012811200320720672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,40,8,128,1,fp8,fp8,0,0.013196800649166108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,40,128,1,float16,float16,0,0.020263999700546265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,40,128,1,float16,fp8,0,0.022644799947738648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,40,128,1,fp8,fp8,0,0.022491200268268584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,1,128,1,float16,fp8,0,0.01252640038728714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,40,2,128,1,fp8,fp8,0,0.014302399754524232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,2,128,1,float16,float16,0,0.012041600048542022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,2,128,1,float16,fp8,0,0.012804800271987915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,2,128,1,fp8,fp8,0,0.012824000418186187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,4,128,1,float16,float16,0,0.01207360029220581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,4,128,1,float16,fp8,0,0.012742400169372559
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,4,128,1,fp8,fp8,0,0.012585599720478059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,40,128,1,float16,float16,0,0.047182399034500125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,8,128,1,float16,float16,0,0.011878400295972823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,8,128,1,float16,fp8,0,0.012856000661849975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,40,128,1,fp8,fp8,0,0.05527359843254089
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,40,128,1,float16,fp8,0,0.0177839994430542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,40,128,1,fp8,fp8,0,0.017790399491786957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,1,128,1,float16,float16,0,0.011591999977827071
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,1,128,1,float16,fp8,0,0.012219200283288956
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,1,128,1,fp8,fp8,0,0.012150400131940842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,2,128,1,float16,float16,0,0.011444800347089768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,2,128,1,float16,fp8,0,0.012244799733161926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,4,128,1,fp8,fp8,0,0.01358879953622818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,4,128,1,float16,float16,0,0.011529599875211715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,4,128,1,float16,fp8,0,0.01241919994354248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,4,128,1,fp8,fp8,0,0.012409599870443344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,8,128,1,float16,float16,0,0.011830399930477142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,8,128,1,float16,fp8,0,0.012689599394798278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,8,128,1,fp8,fp8,0,0.012694400548934937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,40,128,1,float16,float16,0,0.01276479959487915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,1,128,1,float16,float16,0,0.011825600266456604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,40,128,1,float16,fp8,0,0.013489599525928497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,1,128,1,fp8,fp8,0,0.01257600039243698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,40,128,1,fp8,fp8,0,0.013396799564361572
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,1,128,1,float16,float16,0,0.011427199840545655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,1,128,1,float16,fp8,0,0.01218079999089241
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,1,128,1,fp8,fp8,0,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,2,128,1,float16,fp8,0,0.01223519966006279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,2,128,1,fp8,fp8,0,0.01210239976644516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,4,128,1,float16,float16,0,0.011454399675130844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,4,128,1,float16,fp8,0,0.012104000151157378
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,4,128,1,fp8,fp8,0,0.012129600346088409
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,8,128,1,float16,float16,0,0.01149120032787323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,40,128,1,float16,float16,0,0.016387200355529784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,8,128,1,fp8,fp8,0,0.01239520013332367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,40,128,1,float16,float16,0,0.012630400061607362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,40,128,1,float16,fp8,0,0.013475200533866883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,40,2,128,1,float16,float16,0,0.018387199938297273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,40,128,1,fp8,fp8,0,0.013447999954223633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,1,128,1,float16,float16,0,0.011345600336790084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,1,128,1,fp8,fp8,0,0.01204639971256256
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,2,128,1,float16,float16,0,0.011275199800729751
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,40,2,128,1,fp8,fp8,0,0.012241599708795547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,2,128,1,float16,fp8,0,0.01218079999089241
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,2,128,1,fp8,fp8,0,0.012206400185823441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,4,128,1,float16,fp8,0,0.012278400361537933
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,4,128,1,fp8,fp8,0,0.012246400117874146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,8,128,1,float16,float16,0,0.011692799627780914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,8,128,1,float16,fp8,0,0.012452799826860428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,8,128,1,fp8,fp8,0,0.012385600060224534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,2,128,1,float16,float16,0,0.011398400366306304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,40,8,128,1,fp8,fp8,0,0.012700800597667695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,40,8,128,1,float16,fp8,0,0.012328000366687774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,40,1,128,1,fp8,fp8,0,0.013126400113105775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,1,128,1,float16,fp8,0,0.01228799968957901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,40,4,128,1,float16,float16,0,0.011529599875211715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,32,1,128,1,float16,fp8,0,25.57580108642578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,32,1,128,1,fp8,fp8,0,25.579771423339842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,32,2,128,1,float16,fp8,0,25.468714904785156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,32,2,128,1,fp8,fp8,0,25.528169250488283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,32,4,128,1,float16,fp8,0,25.82371826171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,32,1,128,1,float16,float16,0,31.108697509765626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,32,2,128,1,float16,float16,0,31.058718872070312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,32,4,128,1,float16,float16,0,30.57020263671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,32,128,1,float16,float16,0,17.198768615722656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,1,128,1,float16,float16,0,15.826513671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,32,4,128,1,fp8,fp8,0,26.459405517578126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,32,8,128,1,float16,fp8,0,26.52154541015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,32,8,128,1,fp8,fp8,0,26.56187744140625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,32,8,128,1,float16,float16,0,32.05787658691406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,1,128,1,float16,fp8,0,12.883058166503906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,1,128,1,fp8,fp8,0,12.787407684326173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,2,128,1,float16,fp8,0,12.787782287597656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,2,128,1,fp8,fp8,0,12.789631652832032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,2,128,1,float16,float16,0,15.756466674804688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,4,128,1,float16,fp8,0,12.98878173828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,4,128,1,float16,float16,0,15.51302490234375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,4,128,1,fp8,fp8,0,13.168502807617188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,8,128,1,float16,fp8,0,13.501371765136719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,32,128,1,float16,float16,0,8.564228820800782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,8,128,1,fp8,fp8,0,13.1967041015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,8,128,1,float16,float16,0,16.171356201171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,1,128,1,fp8,fp8,0,6.386092758178711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,1,128,1,float16,fp8,0,6.753376007080078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,1,128,1,float16,float16,0,7.8610076904296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,2,128,1,float16,float16,0,7.621858978271485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,2,128,1,float16,fp8,0,6.372564697265625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,2,128,1,fp8,fp8,0,6.436977386474609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,4,128,1,float16,float16,0,7.654345703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,4,128,1,float16,fp8,0,6.415795135498047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,4,128,1,fp8,fp8,0,6.407758331298828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,8,128,1,float16,float16,0,7.929547119140625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,8,128,1,float16,fp8,0,6.4538429260253904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,8,128,1,fp8,fp8,0,6.398822402954101
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,32,128,1,float16,float16,0,4.097889709472656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,1,128,1,float16,float16,0,3.7211265563964844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,1,128,1,float16,fp8,0,3.3801822662353516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,1,128,1,fp8,fp8,0,3.282582473754883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,2,128,1,float16,float16,0,3.801660919189453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,2,128,1,float16,fp8,0,3.1732431411743165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,2,128,1,fp8,fp8,0,3.356145477294922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,4,128,1,float16,float16,0,3.959044647216797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,4,128,1,float16,fp8,0,3.1732431411743165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,4,128,1,fp8,fp8,0,3.3262977600097656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,8,128,1,float16,float16,0,3.7223201751708985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,8,128,1,float16,fp8,0,3.1942352294921874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,8,128,1,fp8,fp8,0,3.353934478759766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,32,1,128,1,float16,fp8,0,14.668646240234375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,32,1,128,1,float16,float16,0,17.528558349609376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,32,1,128,1,fp8,fp8,0,14.917289733886719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,32,2,128,1,float16,float16,0,17.6515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,32,2,128,1,float16,fp8,0,14.812725830078126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,32,2,128,1,fp8,fp8,0,14.580616760253907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,32,4,128,1,float16,float16,0,18.074491882324217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,32,4,128,1,float16,fp8,0,14.737394714355469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,32,4,128,1,fp8,fp8,0,14.627642822265624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,32,8,128,1,float16,float16,0,18.495077514648436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,32,128,1,float16,fp8,0,3.193339157104492
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,32,32,128,1,fp8,fp8,0,3.2044448852539062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,32,128,1,fp8,fp8,0,6.446399688720703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,32,32,128,1,float16,fp8,0,6.510345458984375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,32,128,1,float16,fp8,0,13.2419921875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,32,8,128,1,float16,fp8,0,14.813648986816407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,32,128,1,float16,fp8,0,7.486788940429688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,32,32,128,1,fp8,fp8,0,13.202867126464843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,32,128,1,float16,float16,0,9.141657257080078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,32,8,128,1,fp8,fp8,0,14.638407897949218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,32,128,1,fp8,fp8,0,7.494161224365234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,1,128,1,float16,fp8,0,7.522441864013672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,1,128,1,float16,float16,0,8.764545440673828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,1,128,1,fp8,fp8,0,7.239988708496094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,2,128,1,float16,float16,0,8.702630615234375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,2,128,1,fp8,fp8,0,7.2401885986328125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,2,128,1,float16,fp8,0,7.328510284423828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,4,128,1,float16,float16,0,8.788864135742188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,4,128,1,float16,fp8,0,7.289572906494141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,32,128,1,float16,fp8,0,3.796945571899414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,32,128,1,float16,float16,0,4.525444793701172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,4,128,1,fp8,fp8,0,7.422110748291016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,8,128,1,float16,fp8,0,7.318612670898437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,32,128,1,fp8,fp8,0,3.8233951568603515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,8,128,1,float16,float16,0,8.852094268798828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,32,8,128,1,fp8,fp8,0,7.470203399658203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,1,128,1,float16,float16,0,4.101497650146484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,1,128,1,float16,fp8,0,3.6610286712646483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,1,128,1,fp8,fp8,0,3.694841766357422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,2,128,1,float16,fp8,0,3.599798583984375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,2,128,1,float16,float16,0,4.4048622131347654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,2,128,1,fp8,fp8,0,3.62401123046875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,4,128,1,float16,fp8,0,3.783689498901367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,4,128,1,fp8,fp8,0,3.729230499267578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,32,128,1,float16,float16,0,2.3427312850952147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,8,128,1,float16,fp8,0,3.643743896484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,32,128,1,float16,fp8,0,1.887563133239746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,8,128,1,fp8,fp8,0,3.8018783569335937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,1,128,1,float16,float16,0,2.038447952270508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,32,128,1,fp8,fp8,0,2.151433563232422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,1,128,1,fp8,fp8,0,1.816908836364746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,2,128,1,float16,float16,0,2.11363525390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,2,128,1,float16,fp8,0,2.0154560089111326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,2,128,1,fp8,fp8,0,2.191700744628906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,4,128,1,float16,float16,0,2.2333200454711912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,8,128,1,float16,float16,0,4.340022277832031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,4,128,1,float16,fp8,0,1.7868928909301758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,4,128,1,fp8,fp8,0,2.0240751266479493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,8,128,1,float16,float16,0,1.9571792602539062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,1,128,1,float16,fp8,0,1.937798309326172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,8,128,1,fp8,fp8,0,2.0831823348999023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,32,4,128,1,float16,float16,0,5.046664047241211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,32,8,128,1,float16,fp8,0,2.0217599868774414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,32,1,128,1,float16,fp8,0,10.45050277709961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,32,1,128,1,fp8,fp8,0,10.154129791259766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,32,1,128,1,float16,float16,0,12.263398742675781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,32,2,128,1,float16,fp8,0,10.260646057128906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,32,2,128,1,fp8,fp8,0,10.322853088378906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,32,2,128,1,float16,float16,0,12.437372589111328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,32,4,128,1,float16,float16,0,12.353838348388672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,32,128,1,float16,float16,0,6.581199645996094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,32,4,128,1,float16,fp8,0,10.209397125244141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,32,128,1,float16,fp8,0,5.3359935760498045
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,32,4,128,1,fp8,fp8,0,10.530089569091796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,32,128,1,fp8,fp8,0,5.379651260375977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,32,8,128,1,fp8,fp8,0,10.361876678466796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,32,8,128,1,float16,float16,0,12.766455841064452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,1,128,1,float16,fp8,0,5.060876846313477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,1,128,1,fp8,fp8,0,5.070644760131836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,1,128,1,float16,float16,0,6.1673728942871096
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,32,8,128,1,float16,fp8,0,10.423406219482422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,2,128,1,float16,fp8,0,5.150316619873047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,2,128,1,float16,float16,0,6.320374298095703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,2,128,1,fp8,fp8,0,5.1159217834472654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,4,128,1,float16,float16,0,6.198748779296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,4,128,1,float16,fp8,0,5.108687973022461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,4,128,1,fp8,fp8,0,5.111441421508789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,32,128,1,float16,float16,0,3.2391983032226563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,32,128,1,float16,fp8,0,2.6453136444091796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,8,128,1,float16,fp8,0,5.12154541015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,8,128,1,float16,float16,0,6.497110748291016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,32,8,128,1,fp8,fp8,0,5.178822326660156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,32,128,1,fp8,fp8,0,2.763177680969238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,1,128,1,float16,fp8,0,2.55830078125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,1,128,1,fp8,fp8,0,2.6528640747070313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,2,128,1,float16,float16,0,2.9175071716308594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,2,128,1,float16,fp8,0,2.5565792083740235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,2,128,1,fp8,fp8,0,2.7790895462036134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,4,128,1,float16,float16,0,2.739646339416504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,4,128,1,float16,fp8,0,2.5789743423461915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,4,128,1,fp8,fp8,0,2.5659088134765624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,32,128,1,float16,float16,0,1.6004880905151366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,32,128,1,float16,fp8,0,1.5474783897399902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,1,128,1,float16,float16,0,2.687910461425781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,8,128,1,float16,float16,0,2.9782783508300783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,8,128,1,float16,fp8,0,2.829012870788574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,32,128,1,fp8,fp8,0,1.3519743919372558
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,1,128,1,float16,float16,0,1.3592160224914551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,2,128,1,float16,float16,0,1.3432656288146974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,1,128,1,float16,fp8,0,1.4744671821594237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,2,128,1,fp8,fp8,0,1.3224479675292968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,2,128,1,float16,fp8,0,1.4596559524536132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,4,128,1,float16,float16,0,1.3518320083618165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,4,128,1,float16,fp8,0,1.4047087669372558
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,4,128,1,fp8,fp8,0,1.4081711769104004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,8,128,1,float16,fp8,0,1.2800623893737793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,8,128,1,fp8,fp8,0,1.3108976364135743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,32,8,128,1,fp8,fp8,0,2.5592784881591797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,1,128,1,fp8,fp8,0,1.4173359870910645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,32,8,128,1,float16,float16,0,1.4309231758117675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,32,1,128,1,float16,fp8,0,13.312467956542969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,32,1,128,1,fp8,fp8,0,13.522804260253906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,32,2,128,1,float16,fp8,0,13.434477233886719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,32,1,128,1,float16,float16,0,16.30247344970703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,32,2,128,1,fp8,fp8,0,13.452386474609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,32,2,128,1,float16,float16,0,16.283946228027343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,32,4,128,1,float16,float16,0,16.107606506347658
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,32,4,128,1,float16,fp8,0,13.228254699707032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,32,128,1,float16,fp8,0,7.016295623779297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,32,128,1,float16,float16,0,8.499002838134766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,32,4,128,1,fp8,fp8,0,13.346315002441406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,32,128,1,fp8,fp8,0,7.046482849121094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,32,8,128,1,float16,fp8,0,13.60789794921875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,32,8,128,1,fp8,fp8,0,13.391700744628906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,1,128,1,float16,float16,0,7.788655853271484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,32,8,128,1,float16,float16,0,16.451429748535155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,1,128,1,float16,fp8,0,6.666206359863281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,1,128,1,fp8,fp8,0,6.698825836181641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,2,128,1,float16,fp8,0,6.690721893310547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,2,128,1,float16,float16,0,8.589497375488282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,2,128,1,fp8,fp8,0,7.0573982238769535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,4,128,1,float16,fp8,0,6.6097053527832035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,4,128,1,float16,float16,0,8.253593444824219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,4,128,1,fp8,fp8,0,6.797092437744141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,8,128,1,float16,float16,0,8.361225891113282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,8,128,1,float16,fp8,0,7.322764587402344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,32,128,1,fp8,fp8,0,3.4508670806884765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,32,128,1,float16,fp8,0,3.6237728118896486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,32,128,1,float16,float16,0,4.264051055908203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,1,128,1,float16,fp8,0,3.2997886657714846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,1,128,1,float16,float16,0,3.895638275146484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,32,8,128,1,fp8,fp8,0,7.412814331054688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,1,128,1,fp8,fp8,0,3.417969512939453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,2,128,1,float16,float16,0,4.149272155761719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,2,128,1,fp8,fp8,0,3.3853424072265623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,4,128,1,float16,float16,0,3.9544239044189453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,4,128,1,float16,fp8,0,3.329995346069336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,4,128,1,fp8,fp8,0,3.466718292236328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,32,128,1,float16,float16,0,2.0323135375976564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,8,128,1,float16,float16,0,4.066932678222656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,8,128,1,float16,fp8,0,3.3546031951904296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,32,128,1,fp8,fp8,0,1.7251167297363281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,32,128,1,float16,fp8,0,2.034793663024902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,1,128,1,float16,float16,0,1.7876848220825194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,8,128,1,fp8,fp8,0,3.4681983947753907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,32,2,128,1,float16,fp8,0,3.5501968383789064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,1,128,1,float16,fp8,0,1.8150096893310548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,1,128,1,fp8,fp8,0,1.7940847396850585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,2,128,1,float16,fp8,0,1.6534736633300782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,2,128,1,fp8,fp8,0,1.6647247314453124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,4,128,1,float16,fp8,0,1.6607551574707031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,4,128,1,fp8,fp8,0,1.6429424285888672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,4,128,1,float16,float16,0,2.0272144317626952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,32,128,1,float16,float16,0,0.9511088371276856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,8,128,1,float16,float16,0,1.8836591720581055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,8,128,1,float16,fp8,0,1.6869888305664062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,8,128,1,fp8,fp8,0,1.6528816223144531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,32,128,1,fp8,fp8,0,0.8647055625915527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,1,128,1,float16,float16,0,1.0131808280944825
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,1,128,1,float16,fp8,0,0.8828816413879395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,1,128,1,fp8,fp8,0,0.9637056350708008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,2,128,1,float16,float16,0,0.8960639953613281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,2,128,1,fp8,fp8,0,0.823305606842041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,4,128,1,float16,fp8,0,0.8476367950439453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,4,128,1,fp8,fp8,0,0.9153136253356934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,32,2,128,1,float16,float16,0,1.8162160873413087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,8,128,1,float16,float16,0,1.002563190460205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,8,128,1,fp8,fp8,0,0.84193115234375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,32,128,1,float16,fp8,0,0.8751168251037598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,2,128,1,float16,fp8,0,0.8421584129333496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,4,128,1,float16,float16,0,1.0299424171447753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,32,8,128,1,float16,fp8,0,0.8325152397155762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,32,1,128,1,float16,fp8,0,7.65474853515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,32,1,128,1,fp8,fp8,0,7.740409851074219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,32,1,128,1,float16,float16,0,9.23599853515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,32,2,128,1,float16,float16,0,9.09700927734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,32,2,128,1,float16,fp8,0,7.7374320983886715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,32,2,128,1,fp8,fp8,0,7.665414428710937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,32,4,128,1,float16,fp8,0,7.6225837707519535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,32,4,128,1,float16,float16,0,9.114189147949219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,32,128,1,float16,float16,0,4.954315185546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,32,128,1,float16,fp8,0,4.0521728515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,32,4,128,1,fp8,fp8,0,7.693502044677734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,32,128,1,fp8,fp8,0,4.041169738769531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,1,128,1,float16,float16,0,4.453887939453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,32,8,128,1,float16,fp8,0,7.864513397216797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,32,8,128,1,float16,float16,0,9.512830352783203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,32,8,128,1,fp8,fp8,0,7.770905303955078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,1,128,1,fp8,fp8,0,3.8118255615234373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,1,128,1,float16,fp8,0,3.9965919494628905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,2,128,1,float16,fp8,0,3.804065704345703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,2,128,1,float16,float16,0,4.458145523071289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,2,128,1,fp8,fp8,0,3.8930591583251952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,4,128,1,fp8,fp8,0,3.8151294708251955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,4,128,1,float16,fp8,0,3.9552143096923826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,4,128,1,float16,float16,0,4.851681518554687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,32,128,1,float16,float16,0,2.4080575942993163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,8,128,1,float16,fp8,0,3.8943103790283202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,8,128,1,float16,float16,0,4.573012924194336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,32,128,1,float16,fp8,0,2.028542327880859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,32,8,128,1,fp8,fp8,0,3.8414207458496095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,32,128,1,fp8,fp8,0,2.0580703735351564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,1,128,1,float16,float16,0,2.3870367050170898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,1,128,1,float16,fp8,0,2.022902488708496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,1,128,1,fp8,fp8,0,1.9129951477050782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,2,128,1,float16,float16,0,2.228001594543457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,2,128,1,float16,fp8,0,1.9051456451416016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,2,128,1,fp8,fp8,0,1.908359909057617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,4,128,1,float16,float16,0,2.2321887969970704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,4,128,1,float16,fp8,0,2.0945472717285156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,4,128,1,fp8,fp8,0,1.92303524017334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,32,128,1,float16,float16,0,1.135319995880127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,8,128,1,float16,float16,0,2.208804893493652
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,32,128,1,float16,fp8,0,1.1988880157470703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,8,128,1,fp8,fp8,0,1.9539375305175781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,32,8,128,1,float16,fp8,0,2.0842559814453123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,1,128,1,float16,float16,0,1.0200688362121582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,1,128,1,float16,fp8,0,1.026916790008545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,1,128,1,fp8,fp8,0,1.0765439987182617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,2,128,1,float16,float16,0,1.0692480087280274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,2,128,1,float16,fp8,0,0.9978528022766113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,2,128,1,fp8,fp8,0,0.9772751808166504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,4,128,1,float16,float16,0,1.0760239601135253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,4,128,1,float16,fp8,0,0.9915295600891113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,4,128,1,fp8,fp8,0,1.0062064170837401
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,32,128,1,float16,float16,0,0.5873680114746094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,32,128,1,float16,fp8,0,0.580728006362915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,8,128,1,float16,fp8,0,0.9626895904541015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,1,128,1,float16,float16,0,0.5137184143066407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,1,128,1,float16,fp8,0,0.48277759552001953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,1,128,1,fp8,fp8,0,0.5259039878845215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,2,128,1,float16,float16,0,0.5166240215301514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,2,128,1,float16,fp8,0,0.48234081268310547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,2,128,1,fp8,fp8,0,0.4818751811981201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,4,128,1,float16,float16,0,0.5253200054168701
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,4,128,1,float16,fp8,0,0.48427839279174806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,4,128,1,fp8,fp8,0,0.47894558906555174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,32,128,1,fp8,fp8,0,1.0306544303894043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,8,128,1,float16,float16,0,0.5232079982757568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,8,128,1,float16,fp8,0,0.5073616027832031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,8,128,1,fp8,fp8,0,0.4877664089202881
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,32,32,128,1,fp8,fp8,0,0.5198751926422119
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,8,128,1,fp8,fp8,0,0.9788368225097657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,32,1,128,1,float16,fp8,0,7.172732543945313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,32,1,128,1,fp8,fp8,0,7.259667205810547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,32,2,128,1,float16,fp8,0,7.11065444946289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,32,1,128,1,float16,float16,0,8.829376220703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,32,2,128,1,float16,float16,0,9.225355529785157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,32,2,128,1,fp8,fp8,0,7.127185821533203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,32,8,128,1,float16,float16,0,1.2035375595092774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,32,128,1,float16,fp8,0,3.897905731201172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,32,128,1,float16,float16,0,4.608286285400391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,32,4,128,1,float16,fp8,0,7.289447784423828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,32,4,128,1,fp8,fp8,0,7.265179443359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,32,8,128,1,float16,fp8,0,7.3674156188964846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,32,8,128,1,float16,float16,0,8.599713897705078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,32,8,128,1,fp8,fp8,0,7.310169219970703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,32,4,128,1,float16,float16,0,8.394518280029297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,1,128,1,float16,fp8,0,3.5897201538085937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,1,128,1,fp8,fp8,0,3.564825439453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,2,128,1,float16,fp8,0,3.7192272186279296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,2,128,1,float16,float16,0,4.3303569793701175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,2,128,1,fp8,fp8,0,3.5627983093261717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,32,128,1,fp8,fp8,0,3.922099304199219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,4,128,1,float16,float16,0,4.027595138549804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,1,128,1,float16,float16,0,4.066966247558594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,4,128,1,float16,fp8,0,3.625012969970703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,4,128,1,fp8,fp8,0,3.5797935485839845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,32,128,1,float16,float16,0,2.1919504165649415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,1,128,1,float16,fp8,0,1.7965391159057618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,32,128,1,float16,fp8,0,2.0645120620727537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,8,128,1,float16,fp8,0,3.7429969787597654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,8,128,1,float16,float16,0,4.17163200378418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,1,128,1,fp8,fp8,0,1.9553567886352539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,2,128,1,float16,float16,0,1.9368400573730469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,2,128,1,float16,fp8,0,1.8227872848510742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,4,128,1,float16,float16,0,2.064057540893555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,32,128,1,fp8,fp8,0,1.9285968780517577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,1,128,1,float16,float16,0,2.0293695449829103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,4,128,1,fp8,fp8,0,1.8970783233642579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,8,128,1,float16,float16,0,2.0180335998535157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,8,128,1,float16,fp8,0,1.9058624267578126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,32,128,1,float16,float16,0,1.0865391731262206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,32,128,1,float16,fp8,0,0.9933487892150878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,32,128,1,fp8,fp8,0,1.081612777709961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,1,128,1,float16,float16,0,0.9538991928100586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,8,128,1,fp8,fp8,0,1.8009967803955078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,2,128,1,fp8,fp8,0,1.7981216430664062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,1,128,1,float16,fp8,0,0.9091391563415527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,1,128,1,fp8,fp8,0,0.8976256370544433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,2,128,1,float16,float16,0,0.9523792266845703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,4,128,1,float16,float16,0,0.9528160095214844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,4,128,1,float16,fp8,0,0.9025296211242676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,32,4,128,1,float16,fp8,0,1.7987152099609376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,4,128,1,fp8,fp8,0,0.9021951675415039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,8,128,1,float16,fp8,0,0.9022607803344727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,8,128,1,float16,float16,0,0.9792192459106446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,32,128,1,float16,float16,0,0.5460383892059326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,32,128,1,float16,fp8,0,0.49216642379760744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,32,128,1,fp8,fp8,0,0.50905442237854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,1,128,1,float16,float16,0,0.47951679229736327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,1,128,1,float16,fp8,0,0.4517104148864746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,1,128,1,fp8,fp8,0,0.46970558166503906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,2,128,1,float16,float16,0,0.48667359352111816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,2,128,1,float16,fp8,0,0.44968161582946775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,2,128,1,fp8,fp8,0,0.4612127780914307
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,4,128,1,float16,float16,0,0.4878575801849365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,4,128,1,float16,fp8,0,0.4510064125061035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,4,128,1,fp8,fp8,0,0.45995359420776366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,2,128,1,fp8,fp8,0,0.8993472099304199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,8,128,1,float16,float16,0,0.49251041412353513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,8,128,1,float16,fp8,0,0.4561007976531982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,32,8,128,1,fp8,fp8,0,0.4605231761932373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,32,128,1,float16,float16,0,0.27469439506530763
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,32,128,1,float16,fp8,0,0.2591615915298462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,32,128,1,fp8,fp8,0,0.2511023998260498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,32,8,128,1,fp8,fp8,0,3.6207344055175783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,1,128,1,float16,fp8,0,0.23074240684509278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,1,128,1,fp8,fp8,0,0.23636960983276367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,2,128,1,float16,float16,0,0.24512801170349122
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,2,128,1,fp8,fp8,0,0.23092319965362548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,2,128,1,float16,fp8,0,0.23315680027008057
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,4,128,1,float16,float16,0,0.24833440780639648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,4,128,1,fp8,fp8,0,0.23249440193176268
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,8,128,1,float16,float16,0,0.2508575916290283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,8,128,1,float16,fp8,0,0.2369152069091797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,8,128,1,fp8,fp8,0,0.9098496437072754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,8,128,1,fp8,fp8,0,0.23428640365600586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,32,2,128,1,float16,fp8,0,0.9647376060485839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,1,128,1,float16,float16,0,0.2440000057220459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,32,4,128,1,float16,fp8,0,0.2314608097076416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,32,1,128,1,float16,fp8,0,4.217251205444336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,32,1,128,1,fp8,fp8,0,4.2159679412841795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,32,1,128,1,float16,float16,0,4.673463821411133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,32,2,128,1,float16,float16,0,4.740927886962891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,32,2,128,1,float16,fp8,0,4.21099853515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,32,2,128,1,fp8,fp8,0,4.223072052001953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,32,4,128,1,float16,float16,0,4.925465774536133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,32,4,128,1,fp8,fp8,0,4.237233734130859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,32,8,128,1,float16,fp8,0,4.359262466430664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,32,8,128,1,fp8,fp8,0,4.332366561889648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,32,8,128,1,float16,float16,0,5.069900894165039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,32,128,1,float16,fp8,0,2.3263456344604494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,32,128,1,fp8,fp8,0,2.4466175079345702
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,1,128,1,float16,float16,0,2.3027568817138673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,32,4,128,1,float16,fp8,0,4.233798217773438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,1,128,1,float16,fp8,0,2.113390350341797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,1,128,1,fp8,fp8,0,2.1129823684692384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,2,128,1,float16,fp8,0,2.106608009338379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,2,128,1,float16,float16,0,2.4502063751220704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,2,128,1,fp8,fp8,0,2.118987274169922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,32,128,1,float16,float16,0,2.7016063690185548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,4,128,1,float16,fp8,0,2.1045600891113283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,4,128,1,fp8,fp8,0,2.1094064712524414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,8,128,1,float16,float16,0,2.3916704177856447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,32,128,1,float16,float16,0,1.3413408279418946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,32,128,1,float16,fp8,0,1.177451229095459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,8,128,1,fp8,fp8,0,2.2633760452270506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,1,128,1,float16,float16,0,1.1048175811767578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,32,128,1,fp8,fp8,0,1.1713456153869628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,1,128,1,float16,fp8,0,1.1927776336669922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,2,128,1,float16,fp8,0,1.0625103950500487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,4,128,1,float16,float16,0,1.1228879928588866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,4,128,1,float16,fp8,0,1.0921903610229493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,4,128,1,fp8,fp8,0,1.058886432647705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,4,128,1,float16,float16,0,2.497742462158203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,8,128,1,float16,float16,0,1.221059226989746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,32,8,128,1,float16,fp8,0,2.128455924987793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,8,128,1,float16,fp8,0,1.087067222595215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,32,128,1,float16,float16,0,0.6648143768310547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,32,128,1,float16,fp8,0,0.6717887878417969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,32,128,1,fp8,fp8,0,0.5961264133453369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,1,128,1,float16,fp8,0,0.5347583770751954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,1,128,1,fp8,fp8,0,0.6005504131317139
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,1,128,1,fp8,fp8,0,1.1292559623718261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,2,128,1,float16,float16,0,0.5603968143463135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,2,128,1,float16,fp8,0,0.5346367835998536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,2,128,1,fp8,fp8,0,0.546888017654419
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,2,128,1,fp8,fp8,0,1.1487615585327149
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,4,128,1,float16,fp8,0,0.5535632133483886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,4,128,1,fp8,fp8,0,0.5370319843292236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,8,128,1,float16,float16,0,0.580078411102295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,8,128,1,float16,fp8,0,0.5455920219421386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,8,128,1,fp8,fp8,0,0.5496831893920898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,32,128,1,float16,fp8,0,0.30424959659576417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,32,128,1,fp8,fp8,0,0.3037231922149658
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,1,128,1,float16,float16,0,0.285152006149292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,1,128,1,float16,fp8,0,0.2713871955871582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,1,128,1,fp8,fp8,0,0.2740303993225098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,2,128,1,float16,float16,0,0.2856384038925171
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,2,128,1,float16,fp8,0,0.27194719314575194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,2,128,1,fp8,fp8,0,0.27405600547790526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,8,128,1,fp8,fp8,0,1.068553638458252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,4,128,1,float16,fp8,0,0.2732352018356323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,4,128,1,fp8,fp8,0,0.2759871959686279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,1,128,1,float16,float16,0,0.5555615901947022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,8,128,1,float16,float16,0,0.2943631887435913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,8,128,1,float16,fp8,0,0.2787631988525391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,8,128,1,fp8,fp8,0,0.2797231912612915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,32,128,1,float16,float16,0,0.18021600246429442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,32,128,1,float16,fp8,0,0.1583295941352844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,32,2,128,1,float16,float16,0,1.1143648147583007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,1,128,1,float16,float16,0,0.1479215979576111
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,32,128,1,fp8,fp8,0,0.1587936043739319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,1,128,1,float16,fp8,0,0.1404096007347107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,1,128,1,fp8,fp8,0,0.14048320055007935
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,2,128,1,float16,float16,0,0.14891840219497682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,2,128,1,float16,fp8,0,0.14140959978103637
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,2,128,1,fp8,fp8,0,0.1413472056388855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,4,128,1,float16,float16,0,0.15211039781570435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,4,128,1,float16,fp8,0,0.14105919599533082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,4,128,1,fp8,fp8,0,0.1432464003562927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,8,128,1,float16,float16,0,0.15406559705734252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,8,128,1,float16,fp8,0,0.14530080556869507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,32,8,128,1,fp8,fp8,0,0.14306720495223998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,32,128,1,float16,float16,0,0.3432832002639771
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,32,4,128,1,float16,float16,0,0.2891103982925415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,32,4,128,1,float16,float16,0,0.5697743892669678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,32,1,128,1,float16,fp8,0,4.118547058105468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,32,1,128,1,fp8,fp8,0,4.137505722045899
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,32,1,128,1,float16,float16,0,4.430287933349609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,32,2,128,1,float16,float16,0,4.5851184844970705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,32,2,128,1,float16,fp8,0,4.1211200714111325
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,32,2,128,1,fp8,fp8,0,4.142852783203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,32,128,1,float16,float16,0,2.7145824432373047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,32,4,128,1,float16,float16,0,4.721212768554688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,32,4,128,1,float16,fp8,0,4.1118896484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,32,4,128,1,fp8,fp8,0,4.141347122192383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,32,128,1,float16,fp8,0,2.3612464904785155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,32,8,128,1,fp8,fp8,0,4.176559829711914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,32,8,128,1,float16,fp8,0,4.2638496398925785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,32,8,128,1,float16,float16,0,4.847588729858399
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,32,128,1,fp8,fp8,0,2.4502511978149415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,1,128,1,float16,fp8,0,2.063707160949707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,1,128,1,float16,float16,0,2.301188850402832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,1,128,1,fp8,fp8,0,2.063199996948242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,2,128,1,float16,float16,0,2.1293008804321287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,2,128,1,float16,fp8,0,2.0652080535888673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,2,128,1,fp8,fp8,0,2.0690080642700197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,4,128,1,float16,float16,0,2.3112560272216798
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,4,128,1,float16,fp8,0,2.095974349975586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,32,128,1,float16,float16,0,1.3467439651489257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,8,128,1,float16,fp8,0,2.1022239685058595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,32,128,1,fp8,fp8,0,1.256004810333252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,8,128,1,float16,float16,0,2.451553535461426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,8,128,1,fp8,fp8,0,2.2147167205810545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,1,128,1,float16,float16,0,1.0749872207641602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,1,128,1,float16,fp8,0,1.11352481842041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,1,128,1,fp8,fp8,0,1.0389488220214844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,2,128,1,float16,float16,0,1.0892671585083007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,2,128,1,fp8,fp8,0,1.0459952354431152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,2,128,1,float16,fp8,0,1.1518447875976563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,4,128,1,float16,float16,0,1.1515711784362792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,4,128,1,float16,fp8,0,1.0656031608581542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,4,128,1,fp8,fp8,0,1.0458335876464844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,8,128,1,float16,fp8,0,1.0619855880737306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,8,128,1,fp8,fp8,0,1.0997743606567383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,32,128,1,float16,fp8,0,0.603718376159668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,1,128,1,float16,float16,0,0.541812801361084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,32,4,128,1,fp8,fp8,0,2.074140739440918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,1,128,1,float16,fp8,0,0.5305952072143555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,1,128,1,fp8,fp8,0,0.5246367931365967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,2,128,1,float16,float16,0,0.5446080207824707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,2,128,1,float16,fp8,0,0.528172779083252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,2,128,1,fp8,fp8,0,0.5281311988830566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,4,128,1,float16,float16,0,0.556385612487793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,4,128,1,float16,fp8,0,0.5309279918670654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,4,128,1,fp8,fp8,0,0.5322864055633545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,8,128,1,float16,float16,0,0.5709455966949463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,8,128,1,float16,fp8,0,0.533838415145874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,32,128,1,float16,float16,0,0.34297120571136475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,8,128,1,fp8,fp8,0,0.5374207973480225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,32,128,1,fp8,fp8,0,0.3072976112365723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,1,128,1,float16,float16,0,0.27701919078826903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,1,128,1,float16,fp8,0,0.26935999393463134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,1,128,1,fp8,fp8,0,0.2679327964782715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,2,128,1,float16,float16,0,0.27839839458465576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,2,128,1,float16,fp8,0,0.2697695970535278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,2,128,1,fp8,fp8,0,0.2682960033416748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,4,128,1,float16,float16,0,0.28354079723358155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,4,128,1,float16,fp8,0,0.2713920116424561
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,32,128,1,fp8,fp8,0,0.5993055820465087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,8,128,1,float16,float16,0,0.2906352043151855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,8,128,1,float16,fp8,0,0.2750175952911377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,32,128,1,float16,fp8,0,1.1815679550170899
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,8,128,1,fp8,fp8,0,0.27432639598846437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,32,128,1,float16,fp8,0,0.16142239570617675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,1,128,1,float16,float16,0,0.14442880153656007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,1,128,1,float16,fp8,0,0.14002879858016967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,1,128,1,fp8,fp8,0,0.13947199583053588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,2,128,1,float16,float16,0,0.14513440132141114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,2,128,1,float16,fp8,0,0.140283203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,2,128,1,fp8,fp8,0,0.14022400379180908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,4,128,1,float16,float16,0,0.1475551962852478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,4,128,1,float16,fp8,0,0.1409119963645935
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,4,128,1,fp8,fp8,0,0.14094079732894899
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,8,128,1,float16,float16,0,0.15179840326309205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,32,128,1,float16,fp8,0,0.3094223976135254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,8,128,1,float16,fp8,0,0.14256800413131715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,8,128,1,fp8,fp8,0,0.1434208035469055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,32,128,1,float16,float16,0,0.09469599723815918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,32,128,1,float16,fp8,0,0.08681120276451111
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,32,128,1,fp8,fp8,0,0.08638240098953247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,1,128,1,float16,float16,0,0.07449759840965271
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,1,128,1,float16,fp8,0,0.07191519737243653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,1,128,1,fp8,fp8,0,0.07207840085029601
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,2,128,1,float16,float16,0,0.07534239888191223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,2,128,1,float16,fp8,0,0.07209759950637817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,2,128,1,fp8,fp8,0,0.07150080204010009
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,4,128,1,float16,float16,0,0.07764959931373597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,4,128,1,float16,fp8,0,0.07211679816246033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,32,32,128,1,float16,float16,0,0.699403190612793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,4,128,1,fp8,fp8,0,0.07267199754714966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,8,128,1,float16,float16,0,0.08208000063896179
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,8,128,1,float16,fp8,0,0.07344959974288941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,32,8,128,1,fp8,fp8,0,0.07315199971199035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,32,128,1,float16,float16,0,0.1782271981239319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,32,32,128,1,fp8,fp8,0,0.16058720350265504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,32,1,128,1,float16,float16,0,2.6783647537231445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,32,1,128,1,float16,fp8,0,2.5507856369018556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,32,1,128,1,fp8,fp8,0,2.5436864852905274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,32,2,128,1,float16,float16,0,2.6569488525390623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,32,8,128,1,float16,float16,0,1.146726417541504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,32,2,128,1,float16,fp8,0,2.523593521118164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,32,4,128,1,fp8,fp8,0,0.2699120044708252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,32,2,128,1,fp8,fp8,0,2.6334447860717773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,32,4,128,1,float16,fp8,0,2.5560911178588865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,32,4,128,1,fp8,fp8,0,2.6714223861694335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,32,8,128,1,float16,fp8,0,2.576919937133789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,32,8,128,1,float16,float16,0,2.858203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,32,8,128,1,fp8,fp8,0,2.5803136825561523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,32,128,1,float16,float16,0,1.6672496795654297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,32,128,1,float16,fp8,0,1.5144000053405762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,32,128,1,fp8,fp8,0,1.51321439743042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,1,128,1,float16,fp8,0,1.269472026824951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,1,128,1,fp8,fp8,0,1.3370320320129394
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,2,128,1,float16,float16,0,1.305668830871582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,2,128,1,float16,fp8,0,1.3466431617736816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,2,128,1,fp8,fp8,0,1.2889792442321777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,4,128,1,float16,float16,0,1.3520416259765624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,4,128,1,float16,fp8,0,1.2851712226867675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,4,128,1,fp8,fp8,0,1.2938976287841797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,32,4,128,1,float16,float16,0,2.6447839736938477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,8,128,1,float16,float16,0,1.4439423561096192
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,32,128,1,float16,float16,0,0.8933600425720215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,1,128,1,float16,float16,0,0.6528031826019287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,32,128,1,fp8,fp8,0,0.7644768238067627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,1,128,1,float16,fp8,0,0.6874176025390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,8,128,1,fp8,fp8,0,1.3169903755187988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,1,128,1,fp8,fp8,0,0.695249605178833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,2,128,1,float16,fp8,0,0.648799991607666
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,2,128,1,fp8,fp8,0,0.6740960121154785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,4,128,1,float16,float16,0,0.6746047973632813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,4,128,1,float16,fp8,0,0.6582047939300537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,4,128,1,fp8,fp8,0,0.6532527923583984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,8,128,1,float16,float16,0,0.699121618270874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,8,128,1,float16,fp8,0,0.6647344112396241
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,32,128,1,float16,float16,0,0.42916479110717776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,32,128,1,float16,fp8,0,0.4081791877746582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,32,128,1,fp8,fp8,0,0.39065120220184324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,1,128,1,float16,float16,0,0.33310399055480955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,1,128,1,float16,fp8,0,0.3410144090652466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,8,128,1,float16,fp8,0,1.334068775177002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,2,128,1,float16,float16,0,0.3363744020462036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,2,128,1,float16,fp8,0,0.33872959613800047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,2,128,1,fp8,fp8,0,0.3319135904312134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,32,128,1,float16,fp8,0,0.7597472190856933
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,4,128,1,float16,float16,0,0.3426575899124146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,4,128,1,float16,fp8,0,0.33376638889312743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,4,128,1,fp8,fp8,0,0.3344640016555786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,8,128,1,float16,float16,0,0.3545567989349365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,8,128,1,float16,fp8,0,0.33849759101867677
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,8,128,1,fp8,fp8,0,0.33935999870300293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,32,128,1,float16,fp8,0,0.2030911922454834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,2,128,1,float16,float16,0,0.6586688041687012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,1,128,1,float16,float16,0,0.17380800247192382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,1,128,1,float16,fp8,0,0.17073919773101806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,1,128,1,fp8,fp8,0,0.16990879774093628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,2,128,1,float16,float16,0,0.17544159889221192
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,2,128,1,float16,fp8,0,0.17046400308609008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,2,128,1,fp8,fp8,0,0.17077920436859131
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,4,128,1,float16,float16,0,0.17874560356140137
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,4,128,1,float16,fp8,0,0.17136479616165162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,4,128,1,fp8,fp8,0,0.17141439914703369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,8,128,1,float16,float16,0,0.18474080562591552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,8,128,1,float16,fp8,0,0.17460160255432128
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,8,128,1,fp8,fp8,0,0.17425600290298462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,32,128,1,float16,fp8,0,0.10954079627990723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,32,128,1,fp8,fp8,0,0.10867359638214111
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,1,128,1,float16,float16,0,0.09169120192527772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,32,8,128,1,fp8,fp8,0,0.6674928188323974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,1,128,1,float16,fp8,0,0.08831040263175964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,1,128,1,fp8,fp8,0,0.08749920129776001
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,2,128,1,float16,float16,0,0.09362400174140931
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,2,128,1,float16,fp8,0,0.08919680118560791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,32,1,128,1,fp8,fp8,0,0.33069279193878176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,2,128,1,fp8,fp8,0,0.08892959952354432
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,4,128,1,float16,float16,0,0.09489439725875855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,4,128,1,float16,fp8,0,0.09142240285873413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,8,128,1,float16,float16,0,0.09809759855270386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,8,128,1,float16,fp8,0,0.09258080124855042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,8,128,1,fp8,fp8,0,0.09300640225410461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,32,128,1,float16,float16,0,0.0652671992778778
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,32,128,1,float16,fp8,0,0.05955359935760498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,32,128,1,fp8,fp8,0,0.058815997838973996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,1,128,1,float16,fp8,0,0.048625600337982175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,1,128,1,fp8,fp8,0,0.04790399968624115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,2,128,1,float16,float16,0,0.04914720058441162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,32,128,1,float16,float16,0,0.22404000759124756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,2,128,1,float16,fp8,0,0.04803040027618408
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,32,32,128,1,fp8,fp8,0,0.2031935930252075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,2,128,1,fp8,fp8,0,0.04862880110740662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,32,1,128,1,float16,float16,0,1.3154543876647948
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,4,128,1,float16,fp8,0,0.048790401220321654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,8,128,1,float16,float16,0,0.05157279968261719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,8,128,1,fp8,fp8,0,0.04926080107688904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,32,128,1,float16,float16,0,0.11886399984359741
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,32,4,128,1,fp8,fp8,0,0.09021440148353577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,1,128,1,float16,float16,0,0.049516800045967105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,32,1,128,1,float16,float16,0,2.6495216369628904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,4,128,1,float16,float16,0,0.04975039958953857
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,4,128,1,fp8,fp8,0,0.048956799507141116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,32,8,128,1,float16,fp8,0,0.049158400297164916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,32,1,128,1,fp8,fp8,0,2.660563278198242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,32,2,128,1,float16,float16,0,2.7165103912353517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,32,2,128,1,float16,fp8,0,2.720568084716797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,32,2,128,1,fp8,fp8,0,2.654408073425293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,32,4,128,1,float16,float16,0,2.8420591354370117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,32,4,128,1,float16,fp8,0,2.6864112854003905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,32,4,128,1,fp8,fp8,0,2.659324836730957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,32,1,128,1,float16,fp8,0,2.6719839096069338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,32,8,128,1,float16,float16,0,2.9999263763427733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,32,128,1,float16,float16,0,1.8250944137573242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,32,8,128,1,float16,fp8,0,2.8130447387695314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,32,128,1,float16,fp8,0,1.7201471328735352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,1,128,1,float16,fp8,0,1.3225296020507813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,32,8,128,1,fp8,fp8,0,2.7536176681518554
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,1,128,1,fp8,fp8,0,1.4810815811157227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,2,128,1,float16,float16,0,1.3796719551086425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,2,128,1,float16,fp8,0,1.3347151756286622
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,2,128,1,fp8,fp8,0,1.3511199951171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,4,128,1,float16,float16,0,1.3718992233276368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,4,128,1,float16,fp8,0,1.356392002105713
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,4,128,1,fp8,fp8,0,1.3558671951293946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,8,128,1,float16,fp8,0,1.3734784126281738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,8,128,1,float16,float16,0,1.4696240425109863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,32,128,1,float16,float16,0,0.9477248191833496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,8,128,1,fp8,fp8,0,1.384228801727295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,32,128,1,fp8,fp8,0,1.6404111862182618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,32,128,1,fp8,fp8,0,0.8305279731750488
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,1,128,1,float16,float16,0,0.6723104000091553
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,1,128,1,fp8,fp8,0,0.695147180557251
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,2,128,1,fp8,fp8,0,0.6773920059204102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,4,128,1,float16,float16,0,0.695140790939331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,4,128,1,float16,fp8,0,0.6886144161224366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,4,128,1,fp8,fp8,0,0.6834176063537598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,8,128,1,float16,float16,0,0.7254255771636963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,8,128,1,float16,fp8,0,0.7026703834533692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,32,128,1,float16,float16,0,0.4730959892272949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,8,128,1,fp8,fp8,0,0.6995151996612549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,32,128,1,float16,fp8,0,0.8317647933959961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,1,128,1,float16,float16,0,0.3449455976486206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,32,128,1,fp8,fp8,0,0.4246880054473877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,1,128,1,float16,fp8,0,0.34416320323944094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,1,128,1,fp8,fp8,0,0.34465599060058594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,1,128,1,float16,fp8,0,0.7160304069519043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,2,128,1,float16,float16,0,0.6828752040863038
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,2,128,1,float16,fp8,0,0.34427518844604493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,2,128,1,fp8,fp8,0,0.34483680725097654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,4,128,1,float16,float16,0,0.35646240711212157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,4,128,1,float16,fp8,0,0.34868319034576417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,4,128,1,fp8,fp8,0,0.3455008029937744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,8,128,1,float16,float16,0,0.3713023900985718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,8,128,1,float16,fp8,0,0.3555039882659912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,32,128,1,float16,float16,0,0.2429424047470093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,8,128,1,fp8,fp8,0,0.35545759201049804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,32,128,1,fp8,fp8,0,0.21758079528808594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,1,128,1,float16,float16,0,0.17599040269851685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,1,128,1,float16,fp8,0,0.1781615972518921
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,1,128,1,fp8,fp8,0,0.17839360237121582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,2,128,1,float16,fp8,0,0.17833919525146485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,2,128,1,fp8,fp8,0,0.17843040227890014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,4,128,1,float16,float16,0,0.18371360301971434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,4,128,1,float16,fp8,0,0.18001439571380615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,32,128,1,float16,fp8,0,0.4220287799835205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,4,128,1,fp8,fp8,0,0.1799631953239441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,8,128,1,float16,float16,0,0.19129600524902343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,8,128,1,float16,fp8,0,0.18359520435333251
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,32,128,1,float16,float16,0,0.1289183974266052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,32,128,1,float16,fp8,0,0.1147487998008728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,32,128,1,fp8,fp8,0,0.11513919830322265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,1,128,1,float16,float16,0,0.09421600103378296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,1,128,1,float16,fp8,0,0.09449120163917542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,1,128,1,fp8,fp8,0,0.09414719939231872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,2,128,1,float16,float16,0,0.09564319849014283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,2,128,1,float16,fp8,0,0.09485120177268982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,2,128,1,fp8,fp8,0,0.09442560076713562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,4,128,1,float16,float16,0,0.09824960231781006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,32,2,128,1,float16,fp8,0,0.6742080211639404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,4,128,1,float16,fp8,0,0.09558240175247193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,4,128,1,fp8,fp8,0,0.09506080150604249
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,8,128,1,float16,float16,0,0.1019487977027893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,8,128,1,float16,fp8,0,0.09718239903450013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,32,8,128,1,fp8,fp8,0,0.09793760180473328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,32,128,1,float16,float16,0,0.06992160081863404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,32,128,1,float16,fp8,0,0.0633791983127594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,32,128,1,fp8,fp8,0,0.06284800171852112
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,1,128,1,float16,float16,0,0.04986720085144043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,1,128,1,float16,fp8,0,0.04960159957408905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,2,128,1,float16,float16,0,0.17866400480270386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,1,128,1,fp8,fp8,0,0.04996159970760346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,2,128,1,float16,float16,0,0.04999839961528778
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,2,128,1,float16,fp8,0,0.04928480088710785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,2,128,1,fp8,fp8,0,0.05062239766120911
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,4,128,1,float16,float16,0,0.051070398092269896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,4,128,1,float16,fp8,0,0.049777600169181827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,4,128,1,fp8,fp8,0,0.05060960054397583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,8,128,1,float16,float16,0,0.05560320019721985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,8,128,1,float16,fp8,0,0.05062400102615357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,32,128,1,float16,float16,0,0.03919360041618347
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,32,128,1,float16,fp8,0,0.032839998602867126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,32,128,1,fp8,fp8,0,0.0323168009519577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,1,128,1,float16,float16,0,0.02884959876537323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,1,128,1,float16,fp8,0,0.02942880094051361
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,1,128,1,fp8,fp8,0,0.029686400294303895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,8,128,1,fp8,fp8,0,0.18431199789047242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,2,128,1,float16,float16,0,0.02905600070953369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,2,128,1,fp8,fp8,0,0.02975200116634369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,4,128,1,float16,fp8,0,0.030129599571228027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,4,128,1,fp8,fp8,0,0.02999520003795624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,8,128,1,float16,float16,0,0.030028799176216127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,32,1,128,1,float16,float16,0,1.3258527755737304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,8,128,1,float16,fp8,0,0.030015999078750612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,8,128,1,fp8,fp8,0,0.030129599571228027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,32,32,128,1,float16,fp8,0,0.2179487943649292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,32,1,128,1,float16,float16,0,1.861017608642578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,32,1,128,1,float16,fp8,0,1.9116144180297852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,32,8,128,1,fp8,fp8,0,0.051177597045898436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,2,128,1,float16,fp8,0,0.02959359884262085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,32,4,128,1,float16,float16,0,0.029345598816871644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,32,1,128,1,fp8,fp8,0,1.9248655319213868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,32,2,128,1,float16,float16,0,0.34796159267425536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,32,2,128,1,float16,float16,0,1.8875680923461915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,32,2,128,1,float16,fp8,0,1.9131248474121094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,32,2,128,1,fp8,fp8,0,1.9267168045043945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,32,4,128,1,float16,float16,0,1.9543216705322266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,32,4,128,1,float16,fp8,0,1.9534927368164063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,32,4,128,1,fp8,fp8,0,1.9486848831176757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,32,8,128,1,float16,float16,0,2.0812816619873047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,32,8,128,1,float16,fp8,0,1.9989248275756837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,32,128,1,float16,float16,0,1.4335696220397949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,32,128,1,float16,fp8,0,1.2989184379577636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,1,128,1,float16,float16,0,0.9376688003540039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,32,8,128,1,fp8,fp8,0,2.0370912551879883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,1,128,1,float16,fp8,0,0.9674991607666016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,32,128,1,fp8,fp8,0,1.3124719619750977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,1,128,1,fp8,fp8,0,0.9760767936706543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,2,128,1,float16,float16,0,0.9441056251525879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,2,128,1,fp8,fp8,0,0.9619711875915528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,4,128,1,float16,float16,0,0.9827535629272461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,4,128,1,float16,fp8,0,0.9888367652893066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,4,128,1,fp8,fp8,0,0.981606388092041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,8,128,1,float16,float16,0,1.0523679733276368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,8,128,1,float16,fp8,0,1.0190640449523927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,8,128,1,fp8,fp8,0,1.006180763244629
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,32,128,1,float16,fp8,0,0.6561903953552246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,1,128,1,float16,float16,0,0.4756159782409668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,32,128,1,fp8,fp8,0,0.6588736057281495
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,1,128,1,float16,fp8,0,0.4942607879638672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,1,128,1,fp8,fp8,0,0.4941103935241699
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,2,128,1,float16,float16,0,0.4787439823150635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,2,128,1,float16,fp8,0,0.49045438766479493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,2,128,1,fp8,fp8,0,0.49201440811157227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,4,128,1,float16,float16,0,0.5008543968200684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,4,128,1,float16,fp8,0,0.5008255958557128
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,4,128,1,fp8,fp8,0,0.5008831977844238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,8,128,1,float16,float16,0,0.5269775867462159
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,32,128,1,float16,float16,0,0.37228960990905763
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,8,128,1,float16,fp8,0,0.5116079807281494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,8,128,1,fp8,fp8,0,0.5135951995849609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,32,128,1,float16,fp8,0,0.33747200965881347
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,1,128,1,float16,float16,0,0.2436000108718872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,1,128,1,float16,fp8,0,0.2525279998779297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,2,128,1,float16,float16,0,0.2479775905609131
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,2,128,1,float16,fp8,0,0.253052806854248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,2,128,1,fp8,fp8,0,0.2514127969741821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,4,128,1,float16,float16,0,0.2579360008239746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,4,128,1,float16,fp8,0,0.25636000633239747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,8,128,1,float16,float16,0,0.27188639640808104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,8,128,1,float16,fp8,0,0.2655855894088745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,8,128,1,fp8,fp8,0,0.26276159286499023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,32,128,1,float16,float16,0,0.193123197555542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,32,128,1,float16,fp8,0,0.17539199590682983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,32,128,1,fp8,fp8,0,0.17467360496520995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,1,128,1,float16,float16,0,0.12799520492553712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,1,128,1,float16,fp8,0,0.1314687967300415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,32,32,128,1,float16,float16,0,0.7232912063598633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,2,128,1,float16,float16,0,0.1299888014793396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,2,128,1,fp8,fp8,0,0.13168799877166748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,4,128,1,float16,float16,0,0.13442879915237427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,4,128,1,float16,fp8,0,0.13384000062942505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,4,128,1,fp8,fp8,0,0.13348480463027954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,8,128,1,float16,float16,0,0.1422320008277893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,8,128,1,float16,fp8,0,0.13786720037460326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,32,128,1,fp8,fp8,0,0.3368736028671265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,8,128,1,fp8,fp8,0,0.13748799562454223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,1,128,1,fp8,fp8,0,0.2507695913314819
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,32,128,1,float16,float16,0,0.1036687970161438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,32,128,1,float16,fp8,0,0.09414880275726319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,1,128,1,float16,float16,0,0.07115839719772339
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,1,128,1,float16,fp8,0,0.07124800086021424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,32,4,128,1,fp8,fp8,0,0.2569263935089111
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,2,128,1,float16,float16,0,0.07176480293273926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,2,128,1,float16,fp8,0,0.07272800207138061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,2,128,1,fp8,fp8,0,0.07202399969100952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,32,2,128,1,float16,fp8,0,0.965236759185791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,4,128,1,float16,float16,0,0.07426400184631347
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,4,128,1,float16,fp8,0,0.07254239916801453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,4,128,1,fp8,fp8,0,0.07232000231742859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,8,128,1,float16,float16,0,0.07884479761123657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,8,128,1,float16,fp8,0,0.07463200092315674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,8,128,1,fp8,fp8,0,0.07451999783515931
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,32,128,1,float16,fp8,0,0.05475839972496033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,1,128,1,fp8,fp8,0,0.13116159439086914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,1,128,1,float16,fp8,0,0.0379967987537384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,1,128,1,fp8,fp8,0,0.03815680146217346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,2,128,1,float16,float16,0,0.03736799955368042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,2,128,1,float16,fp8,0,0.03819360136985779
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,2,128,1,fp8,fp8,0,0.037880000472068784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,4,128,1,float16,float16,0,0.039139199256896975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,4,128,1,float16,fp8,0,0.03835040032863617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,4,128,1,fp8,fp8,0,0.03849120140075683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,8,128,1,float16,float16,0,0.04493600130081177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,8,128,1,float16,fp8,0,0.03902879953384399
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,8,128,1,fp8,fp8,0,0.03878880143165588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,32,128,1,float16,float16,0,0.03477280139923096
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,32,128,1,float16,fp8,0,0.030353599786758424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,32,128,1,fp8,fp8,0,0.03023360073566437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,1,128,1,float16,float16,0,0.0224031999707222
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,32,128,1,fp8,fp8,0,0.09409760236740113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,1,128,1,fp8,fp8,0,0.024137599766254424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,2,128,1,float16,float16,0,0.02282399982213974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,2,128,1,float16,fp8,0,0.024128000438213348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,2,128,1,fp8,fp8,0,0.02404800057411194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,4,128,1,float16,fp8,0,0.024241599440574645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,32,128,1,float16,float16,0,0.05872640013694763
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,32,128,1,fp8,fp8,0,0.05429760217666626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,32,1,128,1,float16,float16,0,0.036687999963760376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,8,128,1,float16,fp8,0,0.024451200664043427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,8,128,1,fp8,fp8,0,0.024438400566577912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,32,128,1,float16,fp8,0,0.020500800013542174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,32,128,1,fp8,fp8,0,0.020340800285339355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,32,2,128,1,float16,fp8,0,0.13135039806365967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,1,128,1,float16,float16,0,0.01756319999694824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,1,128,1,float16,fp8,0,0.019113600254058838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,1,128,1,fp8,fp8,0,0.018743999302387238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,2,128,1,float16,float16,0,0.017907199263572694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,2,128,1,float16,fp8,0,0.018755200505256652
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,2,128,1,fp8,fp8,0,0.019166399538517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,4,128,1,float16,float16,0,0.01777759939432144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,4,128,1,float16,fp8,0,0.019414399564266206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,4,128,1,fp8,fp8,0,0.018862399458885192
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,8,128,1,float16,float16,0,0.018380799889564516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,8,128,1,float16,fp8,0,0.019073599576950075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,8,128,1,fp8,fp8,0,0.019494399428367615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,32,1,128,1,float16,float16,0,0.7415215969085693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,32,1,128,1,float16,fp8,0,0.7876895904541016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,1,128,1,float16,fp8,0,0.02414879947900772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,32,1,128,1,fp8,fp8,0,0.07157599925994873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,4,128,1,float16,float16,0,0.022993600368499754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,32,1,128,1,fp8,fp8,0,0.7870368003845215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,32,2,128,1,float16,float16,0,0.7542719841003418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,4,128,1,fp8,fp8,0,0.024135999381542206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,32,2,128,1,float16,fp8,0,0.7837344169616699
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,32,8,128,1,float16,float16,0,0.02348479926586151
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,32,2,128,1,fp8,fp8,0,0.7850160121917724
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,32,32,128,1,float16,float16,0,0.020316800475120543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,32,4,128,1,float16,float16,0,0.7863215923309326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,32,4,128,1,float16,fp8,0,0.8013263702392578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,32,4,128,1,fp8,fp8,0,0.8022512435913086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,32,128,1,float16,float16,0,0.6399919986724854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,32,8,128,1,float16,fp8,0,0.8268239974975586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,32,8,128,1,fp8,fp8,0,0.8317680358886719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,1,128,1,float16,float16,0,0.37820959091186523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,1,128,1,float16,fp8,0,0.4009967803955078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,2,128,1,float16,float16,0,0.3814448118209839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,2,128,1,float16,fp8,0,0.4008495807647705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,2,128,1,fp8,fp8,0,0.4009359836578369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,4,128,1,float16,float16,0,0.4006671905517578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,4,128,1,float16,fp8,0,0.40546398162841796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,4,128,1,fp8,fp8,0,0.4080512046813965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,8,128,1,float16,float16,0,0.4304096221923828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,8,128,1,float16,fp8,0,0.42253761291503905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,32,128,1,float16,float16,0,0.32790560722351075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,32,128,1,float16,fp8,0,0.27917439937591554
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,32,128,1,fp8,fp8,0,0.27939519882202146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,1,128,1,float16,float16,0,0.19608960151672364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,1,128,1,float16,fp8,0,0.20691680908203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,2,128,1,float16,float16,0,0.19972959756851197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,2,128,1,float16,fp8,0,0.20742559432983398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,32,8,128,1,float16,float16,0,0.8472720146179199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,32,128,1,float16,fp8,0,0.5417039871215821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,32,128,1,fp8,fp8,0,0.543839979171753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,4,128,1,float16,float16,0,0.20738399028778076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,4,128,1,float16,fp8,0,0.2099695920944214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,1,128,1,fp8,fp8,0,0.3979199886322021
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,4,128,1,fp8,fp8,0,0.21099998950958251
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,8,128,1,float16,float16,0,0.22232000827789306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,8,128,1,fp8,fp8,0,0.21810240745544435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,32,128,1,float16,float16,0,0.17147680521011352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,32,128,1,fp8,fp8,0,0.14644960165023804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,1,128,1,float16,float16,0,0.10457760095596313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,1,128,1,float16,fp8,0,0.10946079492568969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,1,128,1,fp8,fp8,0,0.11014399528503419
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,32,8,128,1,fp8,fp8,0,0.4206704139709473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,2,128,1,float16,fp8,0,0.11011840105056762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,4,128,1,float16,float16,0,0.11146880388259887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,4,128,1,float16,fp8,0,0.11236799955368042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,4,128,1,fp8,fp8,0,0.11203680038452149
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,1,128,1,fp8,fp8,0,0.20673120021820068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,8,128,1,float16,float16,0,0.1180799961090088
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,8,128,1,float16,fp8,0,0.11655360460281372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,8,128,1,fp8,fp8,0,0.11628479957580566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,32,128,1,float16,float16,0,0.09263520240783692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,32,128,1,float16,fp8,0,0.08029919862747192
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,32,128,1,fp8,fp8,0,0.08019199967384338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,1,128,1,float16,float16,0,0.05881440043449402
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,1,128,1,float16,fp8,0,0.060273599624633786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,1,128,1,fp8,fp8,0,0.06085119843482971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,2,128,1,float16,float16,0,0.05907520055770874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,2,128,1,float16,fp8,0,0.06131680011749267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,2,128,1,fp8,fp8,0,0.06048480272293091
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,4,128,1,float16,float16,0,0.06232320070266724
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,4,128,1,float16,fp8,0,0.061638402938842776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,4,128,1,fp8,fp8,0,0.06206240057945252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,8,128,1,float16,float16,0,0.06583840250968934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,8,128,1,fp8,fp8,0,0.06360960006713867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,8,128,1,float16,fp8,0,0.21779520511627198
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,32,128,1,float16,float16,0,0.053553599119186404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,32,128,1,float16,fp8,0,0.04686239957809448
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,32,128,1,fp8,fp8,0,0.04624800086021423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,1,128,1,float16,float16,0,0.032016000151634215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,1,128,1,float16,fp8,0,0.033102399110794066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,1,128,1,fp8,fp8,0,0.033011201024055484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,2,128,1,float16,fp8,0,0.03285599946975708
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,2,128,1,fp8,fp8,0,0.03306080102920532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,4,128,1,float16,float16,0,0.0343423992395401
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,4,128,1,float16,fp8,0,0.03335680067539215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,4,128,1,fp8,fp8,0,0.03314880132675171
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,8,128,1,float16,float16,0,0.037857601046562196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,8,128,1,float16,fp8,0,0.03383679986000061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,8,128,1,fp8,fp8,0,0.03419840037822723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,32,128,1,float16,fp8,0,0.025084799528121947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,2,128,1,fp8,fp8,0,0.1090656042098999
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,32,128,1,fp8,fp8,0,0.025067201256752013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,1,128,1,float16,float16,0,0.020788800716400147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,1,128,1,float16,fp8,0,0.02189279943704605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,1,128,1,fp8,fp8,0,0.022073599696159362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,2,128,1,float16,float16,0,0.021198399364948273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,2,128,1,float16,fp8,0,0.02211039960384369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,2,128,1,fp8,fp8,0,0.022123199701309205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,4,128,1,float16,float16,0,0.021323199570178985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,4,128,1,float16,fp8,0,0.022198399901390074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,4,128,1,fp8,fp8,0,0.022308799624443054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,8,128,1,float16,float16,0,0.021899199485778807
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,8,128,1,float16,fp8,0,0.022427199780941008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,32,128,1,float16,float16,0,0.017214399576187134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,32,128,1,float16,fp8,0,0.016673600673675536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,32,128,1,fp8,fp8,0,0.016702400147914888
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,1,128,1,float16,float16,0,0.015028800070285796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,1,128,1,float16,fp8,0,0.015355199575424194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,1,128,1,fp8,fp8,0,0.015433600544929505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,2,128,1,float16,float16,0,0.015132799744606018
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,2,128,1,float16,fp8,0,0.015369600057601929
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,2,128,1,fp8,fp8,0,0.015390400588512421
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,4,128,1,float16,float16,0,0.015080000460147857
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,4,128,1,float16,fp8,0,0.015387199819087982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,4,128,1,fp8,fp8,0,0.015268799662590028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,8,128,1,float16,float16,0,0.015113599598407745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,32,128,1,float16,fp8,0,0.1468816041946411
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,8,128,1,float16,fp8,0,0.01563200056552887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,32,8,128,1,fp8,fp8,0,0.015491199493408204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,32,128,1,float16,float16,0,0.015718400478363037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,32,128,1,float16,fp8,0,0.016041600704193117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,32,128,1,fp8,fp8,0,0.015969599783420562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,1,128,1,float16,float16,0,0.014396800100803376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,1,128,1,float16,fp8,0,0.015118399262428283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,1,128,1,fp8,fp8,0,0.015121600031852723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,32,2,128,1,float16,float16,0,0.10559359788894654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,2,128,1,float16,fp8,0,0.015140800178050995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,32,128,1,float16,float16,0,0.030527999997138976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,4,128,1,float16,float16,0,0.014443199336528777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,4,128,1,float16,fp8,0,0.014963200688362122
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,4,128,1,fp8,fp8,0,0.014937600493431092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,8,128,1,float16,float16,0,0.01462240070104599
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,8,128,1,float16,fp8,0,0.01512639969587326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,8,128,1,fp8,fp8,0,0.015131199359893798
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,32,8,128,1,fp8,fp8,0,0.022441600263118745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,32,1,128,1,float16,float16,0,0.3549599885940552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,32,1,128,1,float16,fp8,0,0.3807935953140259
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,32,2,128,1,fp8,fp8,0,0.20713438987731933
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,32,2,128,1,float16,float16,0,0.35966079235076903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,32,2,128,1,float16,fp8,0,0.3832815885543823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,32,8,128,1,float16,fp8,0,0.06391680240631104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,32,2,128,1,fp8,fp8,0,0.38311679363250734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,32,4,128,1,float16,float16,0,0.37335360050201416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,32,2,128,1,float16,float16,0,0.03228000104427338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,32,4,128,1,float16,fp8,0,0.3953423976898193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,32,4,128,1,fp8,fp8,0,0.39288160800933836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,32,8,128,1,float16,float16,0,0.40120320320129393
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,2,128,1,float16,float16,0,0.014433600008487701
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,32,8,128,1,float16,fp8,0,0.4156623840332031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,32,8,128,1,fp8,fp8,0,0.418990421295166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,32,128,1,float16,float16,0,0.3148960113525391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,32,128,1,float16,fp8,0,0.27720320224761963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,32,128,1,fp8,fp8,0,0.2746527910232544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,1,128,1,float16,fp8,0,0.19564640522003174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,1,128,1,fp8,fp8,0,0.19549920558929443
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,2,128,1,float16,float16,0,0.18537119626998902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,2,128,1,float16,fp8,0,0.19521119594573974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,2,128,1,fp8,fp8,0,0.19741439819335938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,4,128,1,float16,float16,0,0.19444479942321777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,4,128,1,float16,fp8,0,0.20232160091400148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,4,128,1,fp8,fp8,0,0.2002000093460083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,8,128,1,float16,float16,0,0.2091615915298462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,8,128,1,float16,fp8,0,0.21565439701080322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,8,128,1,fp8,fp8,0,0.21653120517730712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,32,128,1,float16,fp8,0,0.1446879982948303
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,32,128,1,fp8,fp8,0,0.1442095994949341
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,32,1,128,1,fp8,fp8,0,0.3802880048751831
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,1,128,1,fp8,fp8,0,0.10574719905853272
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,2,128,1,float16,float16,0,0.09988800287246705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,2,128,1,fp8,fp8,0,0.1060752034187317
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,4,128,1,float16,float16,0,0.10355520248413086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,4,128,1,float16,fp8,0,0.10822240114212037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,4,128,1,fp8,fp8,0,0.10789599418640136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,8,128,1,float16,float16,0,0.10936959981918334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,8,128,1,fp8,fp8,0,0.11336319446563721
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,32,128,1,float16,float16,0,0.08920480012893676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,32,128,1,float16,fp8,0,0.07741119861602783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,32,1,128,1,float16,float16,0,0.18280479907989503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,32,128,1,fp8,fp8,0,0.07716799974441528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,1,128,1,float16,float16,0,0.05598239898681641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,1,128,1,float16,fp8,0,0.0572704017162323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,1,128,1,fp8,fp8,0,0.05722079873085022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,2,128,1,float16,float16,0,0.0567359983921051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,2,128,1,float16,fp8,0,0.05796639919281006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,2,128,1,fp8,fp8,0,0.05815359950065613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,4,128,1,float16,float16,0,0.059462398290634155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,4,128,1,float16,fp8,0,0.05904319882392883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,4,128,1,fp8,fp8,0,0.0590224027633667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,32,128,1,float16,float16,0,0.16539520025253296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,8,128,1,float16,float16,0,0.06275359988212585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,8,128,1,float16,fp8,0,0.061028802394866945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,32,8,128,1,fp8,fp8,0,0.06091840267181396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,32,128,1,float16,fp8,0,0.04175040125846863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,32,128,1,fp8,fp8,0,0.04137440025806427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,1,128,1,float16,fp8,0,0.029521599411964417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,1,128,1,fp8,fp8,0,0.02951039969921112
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,2,128,1,float16,float16,0,0.028126400709152222
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,2,128,1,float16,fp8,0,0.029542401432991028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,2,128,1,fp8,fp8,0,0.029528000950813295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,4,128,1,float16,float16,0,0.029867199063301087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,4,128,1,float16,fp8,0,0.03015359938144684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,4,128,1,fp8,fp8,0,0.03022879958152771
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,8,128,1,float16,float16,0,0.03599199950695038
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,8,128,1,float16,fp8,0,0.11436959505081176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,8,128,1,fp8,fp8,0,0.030419200658798218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,32,128,1,float16,float16,0,0.028951999545097352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,32,128,1,float16,fp8,0,0.021006399393081666
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,32,2,128,1,fp8,fp8,0,0.01499200016260147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,32,128,1,fp8,fp8,0,0.02107200026512146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,1,128,1,float16,float16,0,0.01785759925842285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,1,128,1,float16,fp8,0,0.018940800428390504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,1,128,1,fp8,fp8,0,0.018932799994945525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,2,128,1,float16,float16,0,0.017985600233078002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,2,128,1,float16,fp8,0,0.019046400487422944
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,2,128,1,fp8,fp8,0,0.018915200233459474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,4,128,1,float16,float16,0,0.01857600063085556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,4,128,1,float16,fp8,0,0.0191648006439209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,4,128,1,fp8,fp8,0,0.019180800020694732
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,8,128,1,float16,float16,0,0.01905920058488846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,8,128,1,float16,fp8,0,0.019307200610637665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,32,8,128,1,fp8,fp8,0,0.019351999461650848
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,32,128,1,float16,fp8,0,0.014843200147151948
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,32,128,1,fp8,fp8,0,0.014819200336933135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,1,128,1,float16,float16,0,0.013076800107955932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,1,128,1,float16,fp8,0,0.013648000359535218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,1,128,1,float16,float16,0,0.09805759787559509
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,1,128,1,fp8,fp8,0,0.013795199990272521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,2,128,1,float16,float16,0,0.012992000579833985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,2,128,1,fp8,fp8,0,0.013670399785041809
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,4,128,1,float16,float16,0,0.013356800377368926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,8,128,1,float16,fp8,0,0.030846399068832398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,4,128,1,float16,fp8,0,0.013886399567127228
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,4,128,1,fp8,fp8,0,0.013687999546527862
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,8,128,1,float16,float16,0,0.013476799428462981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,8,128,1,float16,fp8,0,0.013729600608348847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,8,128,1,fp8,fp8,0,0.013902400434017182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,32,128,1,float16,float16,0,0.014008000493049622
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,32,128,1,float16,fp8,0,0.013988800346851349
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,32,128,1,fp8,fp8,0,0.01385280042886734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,1,128,1,float16,float16,0,0.012695999443531036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,1,128,1,float16,fp8,0,0.01324480026960373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,1,128,1,fp8,fp8,0,0.013284799456596375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,2,128,1,float16,float16,0,0.012544000148773193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,2,128,1,float16,fp8,0,0.013169600069522858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,2,128,1,fp8,fp8,0,0.013142399489879608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,32,128,1,float16,float16,0,0.050607997179031375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,4,128,1,float16,float16,0,0.012694400548934937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,1,128,1,float16,fp8,0,0.10592160224914551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,4,128,1,fp8,fp8,0,0.013126400113105775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,8,128,1,float16,float16,0,0.012878400087356568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,32,2,128,1,float16,fp8,0,0.10612800121307372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,32,128,1,float16,float16,0,0.013550400733947754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,32,128,1,float16,fp8,0,0.013620799779891968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,32,128,1,fp8,fp8,0,0.013388800621032714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,1,128,1,float16,float16,0,0.012355200201272964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,1,128,1,float16,fp8,0,0.012945599853992462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,1,128,1,fp8,fp8,0,0.013142399489879608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,2,128,1,float16,float16,0,0.012567999958992004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,2,128,1,float16,fp8,0,0.013284799456596375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,2,128,1,fp8,fp8,0,0.01329759955406189
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,4,128,1,float16,float16,0,0.012691199779510498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,4,128,1,float16,fp8,0,0.013150399923324585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,4,128,1,fp8,fp8,0,0.013257600367069244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,8,128,1,float16,float16,0,0.012622399628162384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,8,128,1,float16,fp8,0,0.013204799592494964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,32,8,128,1,fp8,fp8,0,0.013073599338531494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,32,1,128,1,float16,float16,0,0.18295040130615234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,32,1,128,1,float16,fp8,0,0.19627679586410524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,32,128,1,float16,float16,0,0.015451200306415558
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,32,1,128,1,fp8,fp8,0,0.19722399711608887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,32,2,128,1,float16,float16,0,0.18599679470062255
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,32,2,128,1,float16,fp8,0,0.19811840057373048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,32,1,128,1,float16,float16,0,0.02788960039615631
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,32,4,128,1,float16,float16,0,0.192631995677948
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,8,128,1,float16,fp8,0,0.013312000036239623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,8,128,1,fp8,fp8,0,0.013368000090122224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,32,4,128,1,fp8,fp8,0,0.2016063928604126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,32,8,128,1,float16,float16,0,0.20728321075439454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,32,8,128,1,float16,fp8,0,0.21264638900756835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,32,8,128,1,fp8,fp8,0,0.21455519199371337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,32,128,1,float16,fp8,0,0.21496961116790772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,32,128,1,fp8,fp8,0,0.21348960399627687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,1,128,1,float16,fp8,0,0.1053231954574585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,1,128,1,fp8,fp8,0,0.10482879877090454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,2,128,1,float16,float16,0,0.09934080243110657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,2,128,1,float16,fp8,0,0.10560959577560425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,2,128,1,fp8,fp8,0,0.10595840215682983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,4,128,1,float16,float16,0,0.10349600315093994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,4,128,1,float16,fp8,0,0.10771360397338867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,4,128,1,fp8,fp8,0,0.10779680013656616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,8,128,1,float16,float16,0,0.11069600582122803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,8,128,1,float16,fp8,0,0.11314560174942016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,8,128,1,fp8,fp8,0,0.113755202293396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,32,128,1,float16,float16,0,0.11891679763793946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,32,128,1,float16,fp8,0,0.11305279731750488
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,1,128,1,float16,float16,0,0.05707839727401733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,1,128,1,float16,fp8,0,0.05752639770507813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,1,128,1,fp8,fp8,0,0.05764639973640442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,2,128,1,float16,float16,0,0.057206398248672484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,2,128,1,float16,fp8,0,0.0583728015422821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,32,2,128,1,float16,fp8,0,0.01371999979019165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,2,128,1,fp8,fp8,0,0.0581279993057251
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,4,128,1,float16,float16,0,0.05958240032196045
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,32,4,128,1,float16,fp8,0,0.2035088062286377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,4,128,1,fp8,fp8,0,0.05931360125541687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,8,128,1,float16,float16,0,0.0636352002620697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,8,128,1,float16,fp8,0,0.061806398630142215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,8,128,1,fp8,fp8,0,0.061363202333450315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,32,128,1,float16,float16,0,0.22253119945526123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,32,128,1,float16,float16,0,0.06555839776992797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,32,128,1,float16,fp8,0,0.05974879860877991
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,32,128,1,fp8,fp8,0,0.0597648024559021
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,32,1,128,1,float16,float16,0,0.09698240160942077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,1,128,1,float16,fp8,0,0.029951998591423036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,1,128,1,fp8,fp8,0,0.0298224002122879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,2,128,1,float16,float16,0,0.028507199883461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,2,128,1,fp8,fp8,0,0.029879999160766602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,4,128,1,float16,float16,0,0.03031040132045746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,4,128,1,float16,fp8,0,0.030188798904418945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,4,128,1,fp8,fp8,0,0.03031519949436188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,8,128,1,float16,float16,0,0.03594239950180054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,8,128,1,float16,fp8,0,0.03079040050506592
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,32,128,1,float16,float16,0,0.03595679998397827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,32,128,1,float16,fp8,0,0.03001280128955841
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,32,128,1,fp8,fp8,0,0.02972959876060486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,1,128,1,float16,float16,0,0.017425599694252013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,1,128,1,float16,fp8,0,0.01879040002822876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,1,128,1,fp8,fp8,0,0.0188511997461319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,32,128,1,fp8,fp8,0,0.11302560567855835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,2,128,1,float16,float16,0,0.017745600640773775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,2,128,1,float16,fp8,0,0.01874080002307892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,2,128,1,fp8,fp8,0,0.018880000710487364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,4,128,1,float16,float16,0,0.01809599995613098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,4,128,1,float16,fp8,0,0.019068799912929535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,4,128,1,fp8,fp8,0,0.018892799317836762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,8,128,1,float16,float16,0,0.01842560023069382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,32,4,128,1,float16,fp8,0,0.013233600556850434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,8,128,1,float16,fp8,0,0.019232000410556793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,32,128,1,float16,float16,0,0.019019199907779692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,32,128,1,float16,fp8,0,0.01926079988479614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,32,128,1,fp8,fp8,0,0.01921280026435852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,1,128,1,float16,float16,0,0.013264000415802002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,1,128,1,float16,fp8,0,0.013889600336551667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,1,128,1,fp8,fp8,0,0.013756799697875976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,1,128,1,float16,float16,0,0.028014400601387025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,2,128,1,float16,float16,0,0.013223999738693237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,2,128,1,float16,fp8,0,0.013831999897956849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,2,128,1,float16,fp8,0,0.02999039888381958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,2,128,1,fp8,fp8,0,0.013766400516033173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,4,128,1,float16,float16,0,0.01324480026960373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,4,128,1,float16,fp8,0,0.013886399567127228
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,4,128,1,fp8,fp8,0,0.013903999328613281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,8,128,1,float16,float16,0,0.013363200426101684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,8,128,1,float16,fp8,0,0.013982400298118591
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,32,128,1,float16,float16,0,0.013545599579811097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,32,128,1,float16,fp8,0,0.013822400569915771
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,32,8,128,1,fp8,fp8,0,0.030640000104904176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,1,128,1,float16,float16,0,0.012438400089740754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,1,128,1,float16,fp8,0,0.013206399977207184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,1,128,1,fp8,fp8,0,0.013327999413013459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,2,128,1,float16,float16,0,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,2,128,1,float16,fp8,0,0.013376000523567199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,2,128,1,fp8,fp8,0,0.013132800161838532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,4,128,1,float16,float16,0,0.012761600315570831
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,4,128,1,float16,fp8,0,0.013251200318336487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,4,128,1,fp8,fp8,0,0.013372799754142762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,8,128,1,float16,float16,0,0.01292639970779419
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,8,128,1,float16,fp8,0,0.013625599443912506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,8,128,1,fp8,fp8,0,0.01342719942331314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,32,128,1,float16,float16,0,0.01310880035161972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,32,128,1,float16,fp8,0,0.013387200236320496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,32,128,1,fp8,fp8,0,0.013478399813175201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,1,128,1,float16,float16,0,0.012460800260305405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,1,128,1,float16,fp8,0,0.013147200644016265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,1,128,1,fp8,fp8,0,0.013038399815559387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,2,128,1,float16,float16,0,0.012356799840927125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,2,128,1,float16,fp8,0,0.013072000443935394
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,2,128,1,fp8,fp8,0,0.013070400059223174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,4,128,1,float16,float16,0,0.012411200255155564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,4,128,1,float16,fp8,0,0.013247999548912048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,4,128,1,fp8,fp8,0,0.012921600043773651
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,8,128,1,float16,float16,0,0.012732799351215362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,8,128,1,float16,fp8,0,0.013249599933624267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,32,8,128,1,fp8,fp8,0,0.01329279989004135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,32,128,1,float16,float16,0,0.012836800515651703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,32,128,1,float16,fp8,0,0.013132800161838532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,32,128,1,fp8,fp8,0,0.013180799782276154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,1,128,1,float16,float16,0,0.01215519979596138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,32,2,128,1,fp8,fp8,0,0.1962448000907898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,1,128,1,float16,fp8,0,0.012827199697494508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,1,128,1,fp8,fp8,0,0.012748800218105316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,2,128,1,float16,fp8,0,0.012707200646400452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,2,128,1,float16,float16,0,0.01213119998574257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,32,4,128,1,float16,fp8,0,0.05944640040397644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,4,128,1,float16,float16,0,0.012148799747228623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,4,128,1,float16,fp8,0,0.01281599998474121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,4,128,1,fp8,fp8,0,0.012675200402736665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,8,128,1,float16,fp8,0,0.01297760009765625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,8,128,1,fp8,fp8,0,0.012643200159072877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,32,1,128,1,float16,float16,0,0.09756960272789002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,32,1,128,1,float16,fp8,0,0.10628479719161987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,32,1,128,1,fp8,fp8,0,0.1062656044960022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,32,2,128,1,float16,float16,0,0.10011520385742187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,32,2,128,1,float16,fp8,0,0.1065168023109436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,32,2,128,1,fp8,fp8,0,0.10697920322418213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,32,4,128,1,float16,float16,0,0.1041983962059021
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,32,8,128,1,fp8,fp8,0,0.013928000628948212
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,32,4,128,1,float16,fp8,0,0.10872000455856323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,32,4,128,1,fp8,fp8,0,0.10884319543838501
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,32,8,128,1,float16,float16,0,0.11136800050735474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,32,8,128,1,fp8,fp8,0,0.11380800008773803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,32,128,1,float16,float16,0,0.17780319452285767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,32,128,1,float16,fp8,0,0.18554879426956178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,32,128,1,fp8,fp8,0,0.18624320030212402
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,1,128,1,float16,fp8,0,0.05857920050621033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,1,128,1,fp8,fp8,0,0.05883520245552063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,2,128,1,float16,float16,0,0.05788480043411255
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,2,128,1,fp8,fp8,0,0.059248000383377075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,4,128,1,float16,float16,0,0.059977602958679196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,4,128,1,float16,fp8,0,0.06065120100975037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,4,128,1,fp8,fp8,0,0.0605567991733551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,8,128,1,float16,float16,0,0.0641424000263214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,2,128,1,fp8,fp8,0,0.012763200700283051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,8,128,1,float16,fp8,0,0.0626255989074707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,32,8,128,1,float16,float16,0,0.012374400347471236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,32,128,1,float16,float16,0,0.09585279822349549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,32,128,1,float16,fp8,0,0.09828479886054993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,32,128,1,fp8,fp8,0,0.09861279726028442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,1,128,1,float16,fp8,0,0.030246400833129884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,1,128,1,fp8,fp8,0,0.0301503986120224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,32,32,128,1,fp8,fp8,0,0.013748799264431
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,32,8,128,1,float16,fp8,0,0.11403520107269287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,2,128,1,float16,float16,0,0.028881600499153136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,2,128,1,float16,fp8,0,0.03022719919681549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,2,128,1,fp8,fp8,0,0.030257600545883178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,4,128,1,float16,float16,0,0.029688000679016113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,4,128,1,float16,fp8,0,0.030451199412345885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,4,128,1,fp8,fp8,0,0.030398398637771606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,8,128,1,float16,float16,0,0.0359935998916626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,1,128,1,float16,float16,0,0.056681597232818605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,8,128,1,float16,fp8,0,0.03076480031013489
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,8,128,1,fp8,fp8,0,0.03167999982833862
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,32,128,1,float16,float16,0,0.05166400074958801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,32,128,1,float16,fp8,0,0.04871839880943298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,32,128,1,fp8,fp8,0,0.04874399900436401
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,1,128,1,float16,float16,0,0.01830720007419586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,1,128,1,fp8,fp8,0,0.01929280012845993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,2,128,1,float16,float16,0,0.018571199476718904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,2,128,1,float16,fp8,0,0.019324800372123717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,2,128,1,fp8,fp8,0,0.01945919990539551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,4,128,1,float16,float16,0,0.018825599551200868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,4,128,1,float16,fp8,0,0.019592000544071196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,4,128,1,fp8,fp8,0,0.019630399346351624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,8,128,1,float16,float16,0,0.019446399807929993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,8,128,1,float16,fp8,0,0.019760000705718993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,8,128,1,fp8,fp8,0,0.01969279944896698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,32,128,1,float16,float16,0,0.026363199949264525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,32,128,1,float16,fp8,0,0.02807680070400238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,32,128,1,fp8,fp8,0,0.028147199749946596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,1,128,1,float16,float16,0,0.012886400520801543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,1,128,1,float16,fp8,0,0.013937599956989288
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,1,128,1,fp8,fp8,0,0.013622400164604188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,2,128,1,float16,float16,0,0.013007999956607818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,8,128,1,fp8,fp8,0,0.062059199810028075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,2,128,1,fp8,fp8,0,0.013729600608348847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,4,128,1,float16,float16,0,0.01313599944114685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,4,128,1,fp8,fp8,0,0.013803200423717498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,32,1,128,1,float16,float16,0,0.028067201375961304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,8,128,1,float16,float16,0,0.013409599661827087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,8,128,1,float16,fp8,0,0.014124800264835358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,8,128,1,fp8,fp8,0,0.01387999951839447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,32,128,1,float16,float16,0,0.0174687996506691
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,32,128,1,fp8,fp8,0,0.01823520064353943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,1,128,1,float16,float16,0,0.012460800260305405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,1,128,1,float16,fp8,0,0.013355199992656708
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,1,128,1,fp8,fp8,0,0.01321759968996048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,2,128,1,float16,float16,0,0.01249760016798973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,2,128,1,float16,fp8,0,0.01318880021572113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,2,128,1,fp8,fp8,0,0.013312000036239623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,4,128,1,float16,float16,0,0.01279519945383072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,4,128,1,float16,fp8,0,0.013630400598049163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,8,128,1,float16,float16,0,0.012902399897575379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,8,128,1,float16,fp8,0,0.013460800051689148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,8,128,1,fp8,fp8,0,0.01332319974899292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,32,2,128,1,float16,fp8,0,0.05931040048599243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,32,128,1,float16,float16,0,0.012960000336170197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,32,128,1,float16,fp8,0,0.013391999900341034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,32,128,1,fp8,fp8,0,0.013273599743843078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,1,128,1,float16,float16,0,0.012273599952459335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,1,128,1,fp8,fp8,0,0.012969599664211273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,2,128,1,float16,float16,0,0.012535999715328216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,2,128,1,float16,fp8,0,0.01329279989004135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,2,128,1,fp8,fp8,0,0.013264000415802002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,4,128,1,float16,float16,0,0.012318400293588638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,32,8,128,1,fp8,fp8,0,0.019131200015544893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,4,128,1,float16,fp8,0,0.013014400005340576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,8,128,1,float16,float16,0,0.01255040019750595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,8,128,1,float16,fp8,0,0.013091200590133667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,4,128,1,float16,fp8,0,0.013782399892807006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,8,128,1,fp8,fp8,0,0.013147200644016265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,32,128,1,float16,float16,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,32,128,1,float16,fp8,0,0.013143999874591828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,32,128,1,fp8,fp8,0,0.013020800054073333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,1,128,1,float16,fp8,0,0.013048000633716583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,32,128,1,float16,fp8,0,0.01836639940738678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,2,128,1,float16,float16,0,0.012708799540996551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,2,128,1,float16,fp8,0,0.012854400277137756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,2,128,1,fp8,fp8,0,0.012824000418186187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,4,128,1,float16,float16,0,0.012127999961376191
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,4,128,1,float16,fp8,0,0.012833599746227265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,4,128,1,fp8,fp8,0,0.012798400223255157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,8,128,1,float16,float16,0,0.012089599668979645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,8,128,1,float16,fp8,0,0.01271039992570877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,8,128,1,fp8,fp8,0,0.012836800515651703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,32,4,128,1,fp8,fp8,0,0.013435199856758118
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,32,128,1,float16,fp8,0,0.013094399869441987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,32,128,1,fp8,fp8,0,0.013107199966907502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,1,128,1,float16,float16,0,0.012019199877977371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,1,128,1,float16,fp8,0,0.012779200077056884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,1,128,1,fp8,fp8,0,0.012772800028324127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,1,128,1,float16,fp8,0,0.013046400249004364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,2,128,1,float16,fp8,0,0.012510399520397186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,2,128,1,fp8,fp8,0,0.012520000338554382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,4,128,1,float16,float16,0,0.012062399834394454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,4,128,1,float16,fp8,0,0.012756800651550293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,4,128,1,fp8,fp8,0,0.012720000743865967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,8,128,1,float16,float16,0,0.012110400199890136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,8,128,1,float16,fp8,0,0.012756800651550293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,32,4,128,1,fp8,fp8,0,0.013257600367069244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,8,128,1,fp8,fp8,0,0.012582400441169738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,32,1,128,1,float16,float16,0,0.05713120102882385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,32,1,128,1,float16,fp8,0,0.05971360206604004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,1,128,1,float16,float16,0,0.012329600006341934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,32,1,128,1,fp8,fp8,0,0.059785598516464235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,32,2,128,1,float16,float16,0,0.058457601070404056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,32,1,128,1,fp8,fp8,0,0.012904000282287598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,32,2,128,1,float16,fp8,0,0.060172802209854125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,32,2,128,1,fp8,fp8,0,0.06047520041465759
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,32,4,128,1,float16,float16,0,0.06070079803466797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,32,4,128,1,float16,fp8,0,0.061572802066802976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,32,4,128,1,fp8,fp8,0,0.06151360273361206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,32,8,128,1,float16,float16,0,0.09493280053138733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,32,8,128,1,float16,fp8,0,0.09976639747619628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,32,8,128,1,fp8,fp8,0,0.09937440156936646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,32,128,1,float16,float16,0,0.156550395488739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,32,128,1,float16,fp8,0,0.17303199768066407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,1,128,1,float16,float16,0,0.029297599196434022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,32,128,1,fp8,fp8,0,0.1735200047492981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,1,128,1,float16,fp8,0,0.031043198704719544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,1,128,1,fp8,fp8,0,0.031518399715423584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,2,128,1,float16,fp8,0,0.03100320100784302
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,32,128,1,float16,float16,0,0.012756800651550293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,4,128,1,float16,float16,0,0.03144960105419159
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,4,128,1,float16,fp8,0,0.03128960132598877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,4,128,1,fp8,fp8,0,0.03136959969997406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,8,128,1,float16,float16,0,0.05208960175514221
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,32,1,128,1,float16,fp8,0,0.01937440037727356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,8,128,1,fp8,fp8,0,0.04972479939460754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,32,128,1,float16,float16,0,0.08404800295829773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,32,128,1,float16,fp8,0,0.08507680296897888
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,32,2,128,1,float16,float16,0,0.012127999961376191
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,32,128,1,fp8,fp8,0,0.08560320138931274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,1,128,1,float16,float16,0,0.018811200559139252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,1,128,1,float16,fp8,0,0.01989919990301132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,2,128,1,float16,fp8,0,0.0199072003364563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,32,2,128,1,float16,fp8,0,0.01356319934129715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,4,128,1,float16,float16,0,0.01902559995651245
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,4,128,1,float16,fp8,0,0.019996799528598785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,4,128,1,fp8,fp8,0,0.02008959949016571
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,8,128,1,float16,float16,0,0.026332798600196838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,8,128,1,float16,fp8,0,0.02893120050430298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,8,128,1,fp8,fp8,0,0.02887519896030426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,32,128,1,float16,float16,0,0.04146240055561066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,32,128,1,float16,fp8,0,0.04656960070133209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,32,128,1,fp8,fp8,0,0.04648799896240234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,1,128,1,float16,float16,0,0.013236799836158752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,1,128,1,float16,fp8,0,0.014161600172519684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,1,128,1,fp8,fp8,0,0.01419519931077957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,2,128,1,float16,float16,0,0.029278400540351867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,2,128,1,float16,float16,0,0.013574400544166565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,2,128,1,fp8,fp8,0,0.03152480125427246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,2,128,1,float16,fp8,0,0.014318400621414184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,4,128,1,float16,float16,0,0.013681599497795105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,4,128,1,float16,fp8,0,0.014443199336528777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,4,128,1,fp8,fp8,0,0.01438080072402954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,8,128,1,float16,float16,0,0.017510400712490083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,32,8,128,1,float16,fp8,0,0.04986880123615265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,8,128,1,float16,fp8,0,0.018505600094795228
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,32,128,1,float16,float16,0,0.024876800179481507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,32,128,1,fp8,fp8,0,0.027225598692893982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,1,128,1,float16,float16,0,0.012363199889659882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,1,128,1,float16,fp8,0,0.013148799538612366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,1,128,1,fp8,fp8,0,0.013579200208187103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,2,128,1,float16,fp8,0,0.013350400328636169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,1,128,1,fp8,fp8,0,0.01992799937725067
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,2,128,1,fp8,fp8,0,0.013463999330997466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,4,128,1,float16,float16,0,0.013048000633716583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,4,128,1,float16,fp8,0,0.013505600392818451
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,2,128,1,fp8,fp8,0,0.02017119973897934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,8,128,1,float16,float16,0,0.012905600666999816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,8,128,1,float16,fp8,0,0.013356800377368926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,8,128,1,fp8,fp8,0,0.013575999438762665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,32,128,1,float16,float16,0,0.016950400173664094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,32,128,1,float16,fp8,0,0.017924800515174866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,32,128,1,fp8,fp8,0,0.017871999740600587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,1,128,1,float16,fp8,0,0.013302400708198547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,1,128,1,fp8,fp8,0,0.01358720064163208
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,2,128,1,float16,float16,0,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,2,128,1,float16,fp8,0,0.013049599528312684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,2,128,1,fp8,fp8,0,0.012998400628566742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,4,128,1,float16,float16,0,0.012398400157690049
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,4,128,1,float16,fp8,0,0.013214400410652161
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,4,128,1,fp8,fp8,0,0.01326719969511032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,8,128,1,float16,float16,0,0.012449599802494049
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,8,128,1,float16,fp8,0,0.013051199913024902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,8,128,1,fp8,fp8,0,0.0131632000207901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,32,128,1,float16,float16,0,0.012852799892425538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,2,128,1,fp8,fp8,0,0.013991999626159667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,32,128,1,fp8,fp8,0,0.013040000200271606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,1,128,1,float16,float16,0,0.0119439996778965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,1,128,1,float16,fp8,0,0.012577599287033081
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,1,128,1,fp8,fp8,0,0.012831999361515046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,2,128,1,float16,float16,0,0.012273599952459335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,2,128,1,float16,fp8,0,0.012814399600028992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,2,128,1,fp8,fp8,0,0.01279360055923462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,4,128,1,float16,float16,0,0.012212800234556198
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,2,128,1,float16,float16,0,0.012411200255155564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,4,128,1,fp8,fp8,0,0.012880000472068786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,32,2,128,1,float16,float16,0,0.01879040002822876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,8,128,1,float16,float16,0,0.01202400028705597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,8,128,1,float16,fp8,0,0.012849600613117218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,4,128,1,fp8,fp8,0,0.013420799374580383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,32,128,1,float16,float16,0,0.012675200402736665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,32,128,1,float16,fp8,0,0.013075199723243714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,32,128,1,fp8,fp8,0,0.013036799430847169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,32,1,128,1,float16,float16,0,0.012415999919176102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,1,128,1,float16,float16,0,0.012179200351238251
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,1,128,1,float16,fp8,0,0.01273919939994812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,1,128,1,fp8,fp8,0,0.012777599692344665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,2,128,1,float16,float16,0,0.012193600088357926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,2,128,1,float16,fp8,0,0.012731200456619263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,2,128,1,fp8,fp8,0,0.012652799487113953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,4,128,1,float16,float16,0,0.011966399848461151
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,4,128,1,float16,fp8,0,0.012665599584579468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,4,128,1,fp8,fp8,0,0.01271360069513321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,8,128,1,float16,float16,0,0.011846400052309036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,8,128,1,float16,fp8,0,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,32,8,128,1,fp8,fp8,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,32,128,1,float16,float16,0,0.0124719999730587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,32,128,1,float16,fp8,0,0.01297920048236847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,32,128,1,fp8,fp8,0,0.013279999792575835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,1,128,1,float16,float16,0,0.011828800290822982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,1,128,1,float16,fp8,0,0.012680000066757202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,1,128,1,fp8,fp8,0,0.012507200241088867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,2,128,1,float16,float16,0,0.011990399658679962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,2,128,1,float16,fp8,0,0.012520000338554382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,2,128,1,fp8,fp8,0,0.01273760050535202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,4,128,1,float16,float16,0,0.011840000003576278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,4,128,1,float16,fp8,0,0.012751999497413635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,4,128,1,fp8,fp8,0,0.012534399330615998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,8,128,1,float16,float16,0,0.011868800222873687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,8,128,1,float16,fp8,0,0.012539200484752655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,32,128,1,float16,fp8,0,0.013180799782276154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,32,1,128,1,float16,float16,0,0.021447999775409697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,32,1,128,1,float16,fp8,0,0.022382399439811705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,32,1,128,1,fp8,fp8,0,0.022969600558280946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,32,2,128,1,float16,float16,0,0.02805599868297577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,32,8,128,1,fp8,fp8,0,0.018768000602722167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,32,2,128,1,fp8,fp8,0,0.03127039968967438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,32,4,128,1,float16,float16,0,0.04307200014591217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,32,4,128,1,float16,fp8,0,0.04919680058956146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,32,4,128,1,fp8,fp8,0,0.049089598655700686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,32,8,128,1,float16,float16,0,0.0712112009525299
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,32,8,128,1,float16,fp8,0,0.08444640040397644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,32,8,128,1,fp8,fp8,0,0.08455359935760498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,32,128,1,float16,float16,0,0.12671999931335448
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,32,128,1,float16,fp8,0,0.155131196975708
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,1,128,1,float16,float16,0,0.014596800506114959
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,8,128,1,fp8,fp8,0,0.012875199317932129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,1,128,1,fp8,fp8,0,0.015500800311565399
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,2,128,1,float16,float16,0,0.018500800430774688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,2,128,1,float16,fp8,0,0.02006399929523468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,2,128,1,fp8,fp8,0,0.01992480009794235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,4,128,1,float16,float16,0,0.025927999615669252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,4,128,1,float16,fp8,0,0.028911998867988585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,4,128,1,fp8,fp8,0,0.02871200144290924
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,8,128,1,float16,float16,0,0.04026559889316559
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,8,128,1,float16,fp8,0,0.046507200598716734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,8,128,1,fp8,fp8,0,0.04650880098342895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,32,128,1,float16,float16,0,0.06960960030555725
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,32,128,1,float16,fp8,0,0.08198400139808655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,32,128,1,fp8,fp8,0,0.08193439841270447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,32,8,128,1,fp8,fp8,0,0.012481600046157837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,1,128,1,float16,float16,0,0.013107199966907502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,1,128,1,fp8,fp8,0,0.014094400405883788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,2,128,1,float16,float16,0,0.013273599743843078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,32,2,128,1,float16,fp8,0,0.031560000777244565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,2,128,1,float16,fp8,0,0.01406559944152832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,32,32,128,1,float16,fp8,0,0.02741599977016449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,4,128,1,float16,float16,0,0.01722240000963211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,4,128,1,float16,fp8,0,0.018675200641155243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,8,128,1,float16,float16,0,0.024476799368858337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,8,128,1,float16,fp8,0,0.02754560112953186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,8,128,1,fp8,fp8,0,0.02751680016517639
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,32,128,1,float16,float16,0,0.040011200308799746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,32,128,1,float16,fp8,0,0.04549440145492554
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,32,4,128,1,float16,fp8,0,0.0127920001745224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,1,128,1,float16,fp8,0,0.013182400166988373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,1,128,1,fp8,fp8,0,0.013104000687599182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,32,128,1,fp8,fp8,0,0.15540800094604493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,2,128,1,float16,float16,0,0.012683199346065521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,32,1,128,1,float16,fp8,0,0.015494400262832641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,2,128,1,float16,fp8,0,0.013369600474834441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,4,128,1,float16,float16,0,0.012763200700283051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,4,128,1,float16,fp8,0,0.013544000685214996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,4,128,1,fp8,fp8,0,0.013708800077438354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,8,128,1,float16,float16,0,0.0165120005607605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,8,128,1,float16,fp8,0,0.01813279986381531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,8,128,1,fp8,fp8,0,0.01788959950208664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,32,128,1,float16,float16,0,0.024684800207614897
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,32,128,1,float16,fp8,0,0.02661119997501373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,32,128,1,fp8,fp8,0,0.027009600400924684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,1,128,1,float16,float16,0,0.011800000071525573
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,1,128,1,float16,fp8,0,0.012895999848842621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,1,128,1,fp8,fp8,0,0.012590399384498597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,2,128,1,float16,float16,0,0.012219200283288956
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,2,128,1,float16,fp8,0,0.012593600153923034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,2,128,1,fp8,fp8,0,0.012878400087356568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,4,128,1,float16,float16,0,0.011868800222873687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,4,128,1,float16,fp8,0,0.012851199507713318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,4,128,1,fp8,fp8,0,0.012614400684833526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,8,128,1,float16,float16,0,0.012216000258922577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,8,128,1,float16,fp8,0,0.012955200672149659
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,32,8,128,1,fp8,fp8,0,0.012958399951457977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,32,128,1,float16,float16,0,0.016756799817085267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,32,128,1,float16,fp8,0,0.01770240068435669
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,32,128,1,fp8,fp8,0,0.017582400143146514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,1,128,1,float16,float16,0,0.011684799939393998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,1,128,1,float16,fp8,0,0.012414400279521943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,1,128,1,float16,fp8,0,0.013967999815940857
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,1,128,1,fp8,fp8,0,0.012417600303888322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,2,128,1,float16,float16,0,0.011840000003576278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,2,128,1,float16,fp8,0,0.012627199292182922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,2,128,1,fp8,fp8,0,0.01268640011548996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,2,128,1,fp8,fp8,0,0.014265599846839904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,4,128,1,float16,fp8,0,0.012720000743865967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,4,128,1,fp8,fp8,0,0.012689599394798278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,32,4,128,1,fp8,fp8,0,0.018680000305175783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,8,128,1,float16,float16,0,0.011963199824094772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,8,128,1,float16,fp8,0,0.012664000689983367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,8,128,1,fp8,fp8,0,0.012732799351215362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,32,128,1,float16,fp8,0,0.013076800107955932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,32,128,1,fp8,fp8,0,0.013131199777126313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,1,128,1,float16,float16,0,0.01146399974822998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,32,128,1,fp8,fp8,0,0.0455808013677597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,1,128,1,fp8,fp8,0,0.012191999703645706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,2,128,1,float16,float16,0,0.011505600064992905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,2,128,1,float16,fp8,0,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,2,128,1,fp8,fp8,0,0.012223999947309494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,4,128,1,float16,float16,0,0.01170239970088005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,4,128,1,float16,fp8,0,0.012256000190973282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,4,128,1,fp8,fp8,0,0.012459199875593185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,8,128,1,float16,float16,0,0.011760000139474869
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,8,128,1,float16,fp8,0,0.012721599638462066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,2,128,1,fp8,fp8,0,0.013371199369430542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,32,128,1,float16,float16,0,0.012385600060224534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,32,128,1,float16,fp8,0,0.012863999605178833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,32,128,1,fp8,fp8,0,0.012812800705432892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,1,128,1,float16,float16,0,0.011371199786663056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,1,128,1,float16,fp8,0,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,1,128,1,fp8,fp8,0,0.012328000366687774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,2,128,1,float16,float16,0,0.011584000289440155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,2,128,1,float16,fp8,0,0.012278400361537933
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,2,128,1,fp8,fp8,0,0.012328000366687774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,4,128,1,float16,float16,0,0.011558400094509124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,4,128,1,float16,fp8,0,0.01233920007944107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,4,128,1,fp8,fp8,0,0.012356799840927125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,8,128,1,float16,float16,0,0.011655999720096588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,8,128,1,float16,fp8,0,0.012347199767827988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,32,8,128,1,fp8,fp8,0,0.01239359974861145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,32,128,1,float16,float16,0,0.012241599708795547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,32,128,1,float16,fp8,0,0.01279360055923462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,32,128,1,fp8,fp8,0,0.01263359934091568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,1,128,1,float16,float16,0,0.011404799669981003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,1,128,1,float16,fp8,0,0.012086399644613267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,1,128,1,fp8,fp8,0,0.012195199728012085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,2,128,1,float16,float16,0,0.011286400258541107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,2,128,1,float16,fp8,0,0.012118399888277055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,2,128,1,fp8,fp8,0,0.012108799815177918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,4,128,1,float16,fp8,0,0.012172800302505494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,4,128,1,fp8,fp8,0,0.012246400117874146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,8,128,1,float16,float16,0,0.011526399850845337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,8,128,1,float16,fp8,0,0.012321600317955017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,8,128,1,fp8,fp8,0,0.012308800220489502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,32,4,128,1,float16,float16,0,0.012051200121641159
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,32,128,1,float16,float16,0,0.01252799928188324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,1,128,1,float16,fp8,0,0.012256000190973282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,32,8,128,1,fp8,fp8,0,0.012563200294971466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,32,4,128,1,float16,float16,0,0.011985599994659424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,32,1,128,1,float16,float16,0,0.012876799702644348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,24,1,128,1,float16,fp8,0,19.157551574707032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,24,1,128,1,fp8,fp8,0,19.304888916015624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,24,2,128,1,float16,fp8,0,19.362834167480468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,24,2,128,1,fp8,fp8,0,19.212858581542967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,24,1,128,1,float16,float16,0,22.831875610351563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,24,2,128,1,float16,float16,0,22.79490966796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,24,4,128,1,float16,fp8,0,19.575364685058595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,24,4,128,1,float16,float16,0,22.948526000976564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,24,128,1,float16,float16,0,12.324703979492188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,1,128,1,float16,float16,0,11.851376342773438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,24,4,128,1,fp8,fp8,0,20.001423645019532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,24,8,128,1,float16,fp8,0,20.72169647216797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,24,8,128,1,fp8,fp8,0,20.05176544189453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,24,8,128,1,float16,float16,0,24.740803527832032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,1,128,1,float16,fp8,0,9.612604522705078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,1,128,1,fp8,fp8,0,9.721054077148438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,2,128,1,float16,float16,0,11.587397003173828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,2,128,1,fp8,fp8,0,9.597055816650391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,4,128,1,float16,fp8,0,9.786203002929687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,2,128,1,float16,fp8,0,9.731436920166015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,4,128,1,float16,float16,0,11.68700942993164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,4,128,1,fp8,fp8,0,9.649953460693359
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,24,128,1,float16,float16,0,5.793804931640625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,8,128,1,float16,fp8,0,9.723358154296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,8,128,1,float16,float16,0,11.70323486328125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,8,128,1,fp8,fp8,0,9.827830505371093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,1,128,1,float16,fp8,0,4.770470428466797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,1,128,1,float16,float16,0,5.686368179321289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,1,128,1,fp8,fp8,0,4.842278289794922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,2,128,1,float16,float16,0,6.013756942749024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,2,128,1,float16,fp8,0,4.834942245483399
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,2,128,1,fp8,fp8,0,4.922260665893555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,4,128,1,float16,float16,0,6.043948745727539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,4,128,1,float16,fp8,0,5.032841491699219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,4,128,1,fp8,fp8,0,4.886891174316406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,8,128,1,float16,float16,0,5.863209533691406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,24,128,1,float16,float16,0,2.816057586669922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,8,128,1,float16,fp8,0,4.927707290649414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,8,128,1,fp8,fp8,0,4.881399917602539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,1,128,1,float16,float16,0,2.819468879699707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,1,128,1,float16,fp8,0,2.3896608352661133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,1,128,1,fp8,fp8,0,2.5132543563842775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,2,128,1,float16,float16,0,2.8281328201293947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,2,128,1,float16,fp8,0,2.3863759994506837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,2,128,1,fp8,fp8,0,2.8107536315917967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,4,128,1,float16,float16,0,2.729342460632324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,4,128,1,float16,fp8,0,2.4299295425415037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,4,128,1,fp8,fp8,0,2.550497627258301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,8,128,1,float16,float16,0,2.9066320419311524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,8,128,1,float16,fp8,0,2.4257856369018556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,8,128,1,fp8,fp8,0,2.6175792694091795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,24,1,128,1,float16,fp8,0,11.115259552001953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,24,1,128,1,float16,float16,0,13.20618133544922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,24,1,128,1,fp8,fp8,0,11.10647201538086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,24,2,128,1,float16,float16,0,13.425422668457031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,24,2,128,1,float16,fp8,0,11.428546905517578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,24,2,128,1,fp8,fp8,0,10.968122863769532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,24,4,128,1,float16,float16,0,13.753488159179687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,24,4,128,1,float16,fp8,0,11.07175521850586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,24,4,128,1,fp8,fp8,0,11.16046371459961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,24,8,128,1,float16,float16,0,13.887771606445312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,24,8,128,1,float16,fp8,0,11.313565063476563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,24,8,128,1,fp8,fp8,0,11.083793640136719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,24,128,1,float16,float16,0,7.267017364501953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,24,128,1,float16,fp8,0,2.4292863845825194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,24,24,128,1,fp8,fp8,0,2.4099119186401365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,24,128,1,float16,fp8,0,4.929116821289062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,24,24,128,1,fp8,fp8,0,4.859334564208984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,24,128,1,float16,fp8,0,5.575584030151367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,24,128,1,fp8,fp8,0,5.665131378173828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,24,128,1,float16,fp8,0,9.841545867919923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,1,128,1,float16,float16,0,6.411592102050781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,1,128,1,float16,fp8,0,5.454840087890625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,1,128,1,fp8,fp8,0,5.441041564941406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,24,24,128,1,fp8,fp8,0,9.9140380859375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,2,128,1,float16,float16,0,6.415067291259765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,2,128,1,float16,fp8,0,5.469688034057617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,2,128,1,fp8,fp8,0,5.501875305175782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,4,128,1,float16,fp8,0,5.5410400390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,4,128,1,float16,float16,0,6.43448486328125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,24,128,1,float16,float16,0,3.3470062255859374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,8,128,1,float16,float16,0,6.700643157958984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,24,128,1,float16,fp8,0,2.846353530883789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,8,128,1,float16,fp8,0,5.575534439086914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,8,128,1,fp8,fp8,0,5.5824542999267575
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,24,128,1,fp8,fp8,0,2.8233407974243163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,1,128,1,float16,float16,0,3.1516096115112306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,24,4,128,1,fp8,fp8,0,5.463547134399414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,1,128,1,float16,fp8,0,2.8150863647460938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,1,128,1,fp8,fp8,0,2.8859136581420897
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,2,128,1,float16,float16,0,2.9388864517211912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,2,128,1,float16,fp8,0,2.7121936798095705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,2,128,1,fp8,fp8,0,2.7238208770751955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,4,128,1,float16,float16,0,3.188915252685547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,4,128,1,float16,fp8,0,2.8497312545776365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,4,128,1,fp8,fp8,0,2.7008335113525392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,24,128,1,float16,float16,0,1.6809152603149413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,24,128,1,float16,fp8,0,1.4305808067321777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,8,128,1,float16,float16,0,3.2466800689697264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,8,128,1,float16,fp8,0,2.754660797119141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,24,128,1,fp8,fp8,0,1.5715087890625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,24,8,128,1,fp8,fp8,0,3.081131172180176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,1,128,1,float16,fp8,0,1.344923210144043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,2,128,1,float16,float16,0,1.4857935905456543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,1,128,1,fp8,fp8,0,1.549135971069336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,2,128,1,float16,fp8,0,1.534068775177002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,2,128,1,fp8,fp8,0,1.3637968063354493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,4,128,1,float16,float16,0,1.500823974609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,4,128,1,float16,fp8,0,1.375883197784424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,4,128,1,fp8,fp8,0,1.3526975631713867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,8,128,1,float16,fp8,0,1.4057168006896972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,8,128,1,float16,float16,0,1.6505199432373048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,8,128,1,fp8,fp8,0,1.3995583534240723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,24,1,128,1,float16,float16,0,1.4221376419067382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,24,1,128,1,float16,fp8,0,7.711756896972656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,24,2,128,1,float16,fp8,0,7.747494506835937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,24,1,128,1,float16,float16,0,9.353345489501953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,24,2,128,1,fp8,fp8,0,7.727528381347656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,24,2,128,1,float16,float16,0,9.143386840820312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,24,4,128,1,float16,fp8,0,7.662347412109375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,24,4,128,1,float16,float16,0,9.393807983398437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,24,1,128,1,fp8,fp8,0,7.742849731445313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,24,128,1,float16,fp8,0,3.9552433013916017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,24,128,1,fp8,fp8,0,4.200276947021484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,24,4,128,1,fp8,fp8,0,7.878214263916016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,1,128,1,float16,float16,0,4.451599884033203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,24,8,128,1,fp8,fp8,0,8.016985321044922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,24,8,128,1,float16,fp8,0,8.002371215820313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,24,128,1,float16,float16,0,4.796169662475586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,1,128,1,float16,fp8,0,3.7903839111328126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,24,8,128,1,float16,float16,0,10.148072052001954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,1,128,1,fp8,fp8,0,3.8046783447265624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,2,128,1,float16,float16,0,4.433081436157226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,2,128,1,float16,fp8,0,4.034878540039062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,2,128,1,fp8,fp8,0,3.826900863647461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,4,128,1,float16,float16,0,4.5761760711669925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,4,128,1,float16,fp8,0,3.8239791870117186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,4,128,1,fp8,fp8,0,4.068511962890625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,8,128,1,float16,float16,0,4.63842887878418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,24,128,1,float16,float16,0,2.3016336441040037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,24,128,1,fp8,fp8,0,1.9961824417114258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,24,128,1,float16,fp8,0,2.174185562133789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,8,128,1,fp8,fp8,0,3.866993713378906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,24,8,128,1,float16,fp8,0,4.258833694458008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,1,128,1,float16,float16,0,2.1577392578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,1,128,1,float16,fp8,0,2.0592111587524413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,1,128,1,fp8,fp8,0,2.0731807708740235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,2,128,1,float16,float16,0,2.140617561340332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,2,128,1,fp8,fp8,0,1.9121631622314452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,4,128,1,float16,float16,0,2.044633674621582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,4,128,1,float16,fp8,0,2.0717567443847655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,4,128,1,fp8,fp8,0,1.945083236694336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,8,128,1,float16,float16,0,2.1852767944335936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,24,128,1,float16,fp8,0,1.0141872406005858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,24,128,1,float16,float16,0,1.2376367568969726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,8,128,1,fp8,fp8,0,1.951487922668457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,24,128,1,fp8,fp8,0,1.0577631950378419
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,1,128,1,float16,float16,0,1.0120736122131349
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,1,128,1,float16,fp8,0,1.0888527870178222
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,2,128,1,float16,float16,0,1.1415599822998046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,2,128,1,fp8,fp8,0,0.9859536170959473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,4,128,1,float16,float16,0,1.0962335586547851
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,2,128,1,float16,fp8,0,1.9503423690795898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,4,128,1,float16,fp8,0,1.0218576431274413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,4,128,1,fp8,fp8,0,1.0795599937438964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,8,128,1,float16,float16,0,1.0528207778930665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,8,128,1,fp8,fp8,0,0.9533120155334472
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,24,8,128,1,float16,fp8,0,2.1053680419921874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,1,128,1,fp8,fp8,0,0.9461647987365722
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,2,128,1,float16,fp8,0,0.9652144432067871
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,24,8,128,1,float16,fp8,0,1.102342414855957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,24,1,128,1,float16,fp8,0,9.958147430419922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,24,1,128,1,fp8,fp8,0,10.095438385009766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,24,1,128,1,float16,float16,0,11.831088256835937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,24,2,128,1,float16,fp8,0,10.04665298461914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,24,2,128,1,fp8,fp8,0,9.871363067626953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,24,2,128,1,float16,float16,0,12.015068817138673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,24,4,128,1,float16,fp8,0,10.085198211669923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,24,4,128,1,float16,float16,0,12.211420440673828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,24,128,1,float16,fp8,0,5.328041458129883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,24,128,1,float16,float16,0,6.236876678466797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,24,128,1,fp8,fp8,0,5.227332687377929
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,24,4,128,1,fp8,fp8,0,10.182575988769532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,1,128,1,float16,float16,0,5.9685615539550785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,24,8,128,1,float16,fp8,0,10.31798095703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,24,8,128,1,fp8,fp8,0,10.201156616210938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,24,8,128,1,float16,float16,0,12.604654693603516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,1,128,1,float16,fp8,0,5.0072990417480465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,1,128,1,fp8,fp8,0,5.015385437011719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,2,128,1,float16,fp8,0,5.06192626953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,2,128,1,float16,float16,0,5.90438232421875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,2,128,1,fp8,fp8,0,5.071524810791016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,4,128,1,float16,fp8,0,5.055398559570312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,4,128,1,float16,float16,0,6.042766571044922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,4,128,1,fp8,fp8,0,5.294267272949218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,24,128,1,float16,fp8,0,2.6304128646850584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,24,128,1,float16,float16,0,3.0862863540649412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,8,128,1,float16,fp8,0,5.0101055145263675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,8,128,1,float16,float16,0,6.161859130859375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,24,128,1,fp8,fp8,0,2.6243663787841798
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,24,8,128,1,fp8,fp8,0,5.065115356445313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,1,128,1,float16,fp8,0,2.4996448516845704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,1,128,1,float16,float16,0,2.9073360443115233
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,1,128,1,fp8,fp8,0,2.4909231185913088
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,2,128,1,float16,fp8,0,2.484878349304199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,2,128,1,float16,float16,0,2.9094528198242187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,2,128,1,fp8,fp8,0,2.6787088394165037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,4,128,1,float16,float16,0,2.923409652709961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,4,128,1,float16,fp8,0,2.535083198547363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,24,128,1,float16,fp8,0,1.3709296226501464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,24,128,1,float16,float16,0,1.534556770324707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,8,128,1,float16,fp8,0,2.540444755554199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,8,128,1,float16,float16,0,3.0838159561157226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,8,128,1,fp8,fp8,0,2.5287647247314453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,24,128,1,fp8,fp8,0,1.521348762512207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,1,128,1,float16,float16,0,1.3979887962341309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,1,128,1,fp8,fp8,0,1.3377519607543946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,2,128,1,float16,float16,0,1.4889663696289062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,2,128,1,float16,fp8,0,1.3507295608520509
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,4,128,1,float16,float16,0,1.4725775718688965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,4,128,1,fp8,fp8,0,1.2965888023376464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,8,128,1,float16,fp8,0,1.2835200309753418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,24,4,128,1,fp8,fp8,0,2.572060775756836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,24,128,1,float16,float16,0,0.7274511814117431
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,8,128,1,fp8,fp8,0,1.5396575927734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,24,128,1,float16,fp8,0,0.6584559917449951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,24,128,1,fp8,fp8,0,0.6608528137207031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,1,128,1,float16,float16,0,0.7374144077301026
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,1,128,1,float16,fp8,0,0.6813471794128418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,1,128,1,float16,fp8,0,1.2950927734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,2,128,1,float16,float16,0,0.6773392200469971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,2,128,1,float16,fp8,0,0.7797455787658691
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,2,128,1,fp8,fp8,0,0.6504576206207275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,4,128,1,float16,float16,0,0.685595178604126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,4,128,1,float16,fp8,0,0.7416175842285156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,4,128,1,fp8,fp8,0,0.6431568145751954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,4,128,1,float16,fp8,0,1.488268756866455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,8,128,1,float16,float16,0,0.6901279926300049
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,8,128,1,float16,fp8,0,0.6826528072357178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,8,128,1,fp8,fp8,0,0.6378496170043946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,8,128,1,float16,float16,0,1.6596271514892578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,24,1,128,1,fp8,fp8,0,0.6204432010650635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,24,2,128,1,fp8,fp8,0,1.3565232276916503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,24,1,128,1,float16,fp8,0,5.8579761505126955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,24,1,128,1,fp8,fp8,0,5.783332824707031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,24,1,128,1,float16,float16,0,6.546609497070312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,24,2,128,1,float16,fp8,0,5.767201614379883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,24,2,128,1,float16,float16,0,6.793428802490235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,24,2,128,1,fp8,fp8,0,5.745684814453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,24,4,128,1,float16,float16,0,6.768532562255859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,24,128,1,float16,float16,0,3.4860225677490235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,24,128,1,float16,fp8,0,3.047564888000488
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,24,4,128,1,float16,fp8,0,5.861070251464843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,24,4,128,1,fp8,fp8,0,5.811342239379883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,24,8,128,1,float16,fp8,0,5.947032165527344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,24,128,1,fp8,fp8,0,3.1291135787963866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,24,8,128,1,float16,float16,0,6.966705322265625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,24,8,128,1,fp8,fp8,0,5.961299133300781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,1,128,1,float16,fp8,0,2.851068878173828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,1,128,1,float16,float16,0,3.381105422973633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,1,128,1,fp8,fp8,0,2.8849584579467775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,2,128,1,float16,float16,0,3.266651153564453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,2,128,1,float16,fp8,0,2.8828256607055662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,2,128,1,fp8,fp8,0,2.8525760650634764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,4,128,1,float16,fp8,0,2.8696016311645507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,4,128,1,float16,float16,0,3.3961807250976563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,24,128,1,float16,fp8,0,1.5177760124206543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,8,128,1,float16,float16,0,3.4729598999023437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,8,128,1,float16,fp8,0,2.8865583419799803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,24,128,1,fp8,fp8,0,1.5293423652648925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,8,128,1,fp8,fp8,0,3.0316991806030273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,1,128,1,float16,float16,0,1.684227180480957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,1,128,1,float16,fp8,0,1.4248720169067384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,1,128,1,fp8,fp8,0,1.5724160194396972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,2,128,1,float16,fp8,0,1.432750415802002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,2,128,1,fp8,fp8,0,1.451251220703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,4,128,1,float16,float16,0,1.6695135116577149
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,24,4,128,1,fp8,fp8,0,2.884440040588379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,4,128,1,fp8,fp8,0,1.4721936225891112
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,24,128,1,float16,float16,0,1.7770608901977538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,8,128,1,float16,float16,0,1.6317104339599608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,24,128,1,float16,float16,0,0.8535776138305664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,8,128,1,float16,fp8,0,1.4552528381347656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,24,128,1,float16,fp8,0,0.7737152099609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,24,128,1,fp8,fp8,0,0.7667967796325683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,8,128,1,fp8,fp8,0,1.633019256591797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,1,128,1,float16,float16,0,0.8487471580505371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,1,128,1,float16,fp8,0,0.7601696014404297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,1,128,1,fp8,fp8,0,0.7212287902832031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,2,128,1,float16,fp8,0,0.7141776084899902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,2,128,1,fp8,fp8,0,0.7482768058776855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,4,128,1,float16,float16,0,0.8504544258117676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,2,128,1,float16,float16,0,1.5530847549438476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,4,128,1,fp8,fp8,0,0.720140790939331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,8,128,1,float16,float16,0,0.7867872238159179
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,24,128,1,float16,float16,0,0.45075201988220215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,24,128,1,float16,fp8,0,0.3912992000579834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,8,128,1,fp8,fp8,0,0.7903488159179688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,24,4,128,1,float16,fp8,0,1.4417759895324707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,24,128,1,fp8,fp8,0,0.4470975875854492
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,1,128,1,float16,float16,0,0.38983519077301027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,1,128,1,fp8,fp8,0,0.37940800189971924
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,2,128,1,float16,float16,0,0.3835200071334839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,2,128,1,fp8,fp8,0,0.3621328115463257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,4,128,1,float16,float16,0,0.39611520767211916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,4,128,1,fp8,fp8,0,0.36522879600524905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,8,128,1,float16,float16,0,0.40121917724609374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,8,128,1,float16,fp8,0,0.3686880111694336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,8,128,1,fp8,fp8,0,0.37748160362243655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,2,128,1,float16,float16,0,0.7722335815429687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,4,128,1,float16,fp8,0,0.7408864021301269
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,24,8,128,1,float16,fp8,0,0.7355264186859131
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,1,128,1,float16,fp8,0,0.36133599281311035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,2,128,1,float16,fp8,0,0.4166431903839111
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,24,4,128,1,float16,fp8,0,0.36838719844818113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,24,1,128,1,float16,fp8,0,5.376036834716797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,24,1,128,1,float16,float16,0,6.197600173950195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,24,1,128,1,fp8,fp8,0,5.351459121704101
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,24,2,128,1,float16,float16,0,6.07695198059082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,24,2,128,1,fp8,fp8,0,5.357940673828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,24,4,128,1,float16,fp8,0,5.362481689453125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,24,4,128,1,float16,float16,0,6.384423828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,24,2,128,1,float16,fp8,0,5.344043350219726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,24,128,1,float16,float16,0,3.3635887145996093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,24,4,128,1,fp8,fp8,0,5.530668640136719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,24,128,1,float16,fp8,0,2.9096704483032227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,24,8,128,1,float16,fp8,0,5.55743522644043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,24,128,1,fp8,fp8,0,3.0239583969116213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,24,8,128,1,float16,float16,0,6.582599639892578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,24,8,128,1,fp8,fp8,0,5.538399887084961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,1,128,1,float16,float16,0,2.8570032119750977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,1,128,1,float16,fp8,0,2.7869247436523437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,2,128,1,float16,float16,0,3.011408042907715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,2,128,1,fp8,fp8,0,2.6979648590087892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,4,128,1,float16,float16,0,3.207459259033203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,4,128,1,float16,fp8,0,2.8183567047119142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,4,128,1,fp8,fp8,0,2.7117536544799803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,1,128,1,fp8,fp8,0,2.666771125793457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,8,128,1,float16,float16,0,3.19366397857666
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,8,128,1,float16,fp8,0,2.746481513977051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,2,128,1,float16,fp8,0,2.690399932861328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,24,128,1,float16,float16,0,1.6665872573852538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,24,128,1,float16,fp8,0,1.4568575859069823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,24,8,128,1,fp8,fp8,0,2.9337696075439452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,24,128,1,fp8,fp8,0,1.4580384254455567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,1,128,1,float16,float16,0,1.543081569671631
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,1,128,1,fp8,fp8,0,1.3359855651855468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,2,128,1,float16,fp8,0,1.3375776290893555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,2,128,1,float16,float16,0,1.5375519752502442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,2,128,1,fp8,fp8,0,1.3460623741149902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,4,128,1,float16,float16,0,1.4587759971618652
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,4,128,1,float16,fp8,0,1.3576607704162598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,4,128,1,fp8,fp8,0,1.4642576217651366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,8,128,1,float16,float16,0,1.5080384254455566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,24,128,1,float16,float16,0,0.8078144073486329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,8,128,1,float16,fp8,0,1.3648112297058106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,24,128,1,float16,fp8,0,0.8241264343261718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,24,128,1,fp8,fp8,0,0.7362512111663818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,8,128,1,fp8,fp8,0,1.5253120422363282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,1,128,1,float16,float16,0,0.7154496192932129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,1,128,1,fp8,fp8,0,0.737388801574707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,2,128,1,float16,float16,0,0.7132656097412109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,2,128,1,float16,fp8,0,0.7115600109100342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,2,128,1,fp8,fp8,0,0.6768320083618165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,4,128,1,float16,float16,0,0.7318431854248046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,4,128,1,float16,fp8,0,0.6981711864471436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,24,1,128,1,float16,fp8,0,1.35556640625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,24,128,1,float16,float16,0,0.41185598373413085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,8,128,1,float16,float16,0,0.7400015830993653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,24,128,1,float16,fp8,0,0.37844319343566896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,8,128,1,fp8,fp8,0,0.6922704219818115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,24,128,1,fp8,fp8,0,0.3744256019592285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,1,128,1,float16,float16,0,0.3623728036880493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,1,128,1,fp8,fp8,0,0.34438400268554686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,2,128,1,float16,float16,0,0.3616192102432251
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,2,128,1,float16,fp8,0,0.34359040260314944
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,2,128,1,fp8,fp8,0,0.35391199588775635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,4,128,1,float16,fp8,0,0.34701759815216066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,4,128,1,fp8,fp8,0,0.3464927911758423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,8,128,1,float16,float16,0,0.3804768085479736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,8,128,1,fp8,fp8,0,0.35150880813598634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,24,128,1,float16,float16,0,0.21398398876190186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,24,128,1,float16,fp8,0,0.19411840438842773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,24,128,1,fp8,fp8,0,0.19573919773101806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,1,128,1,float16,float16,0,0.18591519594192504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,1,128,1,float16,fp8,0,0.1752784013748169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,1,128,1,fp8,fp8,0,0.17477279901504517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,2,128,1,float16,float16,0,0.18725119829177855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,2,128,1,float16,fp8,0,0.1752527952194214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,2,128,1,fp8,fp8,0,0.17573920488357545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,4,128,1,float16,float16,0,0.18945759534835815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,4,128,1,float16,fp8,0,0.17838720083236695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,4,128,1,fp8,fp8,0,0.7079823970794678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,8,128,1,float16,fp8,0,0.6981776237487793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,8,128,1,float16,float16,0,0.19385440349578859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,8,128,1,float16,fp8,0,0.1811519980430603
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,8,128,1,fp8,fp8,0,0.18092639446258546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,1,128,1,float16,fp8,0,0.34721601009368896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,4,128,1,float16,float16,0,0.36428959369659425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,24,8,128,1,float16,fp8,0,0.350435209274292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,24,1,128,1,float16,fp8,0,0.7730751991271972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,24,1,128,1,float16,fp8,0,3.1650415420532227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,24,1,128,1,fp8,fp8,0,3.147995185852051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,24,1,128,1,float16,float16,0,3.5494159698486327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,24,2,128,1,float16,float16,0,3.5336448669433596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,24,2,128,1,float16,fp8,0,3.175694465637207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,24,2,128,1,fp8,fp8,0,3.177499198913574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,24,4,128,1,float16,float16,0,3.6619937896728514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,24,4,128,1,float16,fp8,0,3.211564636230469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,24,4,128,1,fp8,fp8,0,3.2034942626953127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,24,4,128,1,fp8,fp8,0,0.17660800218582154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,24,8,128,1,float16,float16,0,3.8209056854248047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,24,8,128,1,float16,fp8,0,3.3585536956787108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,24,8,128,1,fp8,fp8,0,3.261030578613281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,24,128,1,float16,fp8,0,1.8982799530029297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,24,128,1,fp8,fp8,0,1.7667312622070312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,1,128,1,float16,float16,0,1.6912544250488282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,1,128,1,float16,fp8,0,1.5662528038024903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,1,128,1,fp8,fp8,0,1.7021711349487305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,2,128,1,fp8,fp8,0,1.6021200180053712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,4,128,1,float16,float16,0,1.715902328491211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,4,128,1,fp8,fp8,0,1.5917360305786132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,4,128,1,float16,fp8,0,1.7465744018554688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,8,128,1,float16,float16,0,1.793574333190918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,24,128,1,float16,fp8,0,0.8972816467285156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,8,128,1,float16,fp8,0,1.6436511993408203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,8,128,1,fp8,fp8,0,1.7454431533813477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,24,128,1,fp8,fp8,0,1.0243023872375487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,1,128,1,float16,float16,0,0.8449664115905762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,1,128,1,float16,fp8,0,0.8073424339294434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,2,128,1,float16,fp8,0,1.593939208984375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,1,128,1,fp8,fp8,0,0.812833595275879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,2,128,1,float16,float16,0,0.8586463928222656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,2,128,1,float16,fp8,0,0.8320992469787598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,2,128,1,fp8,fp8,0,0.8041664123535156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,24,128,1,float16,float16,0,2.0190288543701174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,4,128,1,float16,float16,0,0.8548959732055664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,4,128,1,float16,fp8,0,0.8871808052062988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,4,128,1,fp8,fp8,0,0.812492847442627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,8,128,1,float16,float16,0,0.8660271644592286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,8,128,1,float16,fp8,0,0.8281023979187012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,8,128,1,fp8,fp8,0,0.8357808113098144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,24,24,128,1,float16,float16,0,0.9915328025817871
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,24,128,1,fp8,fp8,0,0.4489120006561279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,1,128,1,float16,float16,0,0.42068800926208494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,1,128,1,float16,fp8,0,0.4067728042602539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,1,128,1,fp8,fp8,0,0.4007823944091797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,2,128,1,float16,float16,0,0.4223951816558838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,2,128,1,float16,fp8,0,0.4082352161407471
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,2,128,1,fp8,fp8,0,0.4071216106414795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,4,128,1,float16,float16,0,0.4302671909332275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,4,128,1,float16,fp8,0,0.4126560211181641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,4,128,1,fp8,fp8,0,0.41321439743041993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,8,128,1,float16,float16,0,0.4407695770263672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,8,128,1,float16,fp8,0,0.41857118606567384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,24,128,1,float16,float16,0,0.2625391960144043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,8,128,1,fp8,fp8,0,0.4200287818908691
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,24,128,1,fp8,fp8,0,0.23083360195159913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,1,128,1,float16,float16,0,0.2167504072189331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,1,128,1,float16,fp8,0,0.2090143918991089
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,1,128,1,fp8,fp8,0,0.20857760906219483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,2,128,1,float16,float16,0,0.2190160036087036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,2,128,1,float16,fp8,0,0.2101151943206787
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,2,128,1,fp8,fp8,0,0.20941441059112548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,4,128,1,float16,float16,0,0.2225856065750122
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,4,128,1,float16,fp8,0,0.21230080127716064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,4,128,1,fp8,fp8,0,0.2115247964859009
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,8,128,1,float16,float16,0,0.22854559421539306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,8,128,1,float16,fp8,0,0.21509120464324952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,8,128,1,fp8,fp8,0,0.21610720157623292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,24,128,1,float16,float16,0,0.14074239730834961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,24,128,1,float16,fp8,0,0.12247200012207031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,24,128,1,float16,fp8,0,0.447873592376709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,1,128,1,float16,float16,0,0.1145408034324646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,1,128,1,float16,fp8,0,0.10903040170669556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,1,128,1,fp8,fp8,0,0.10794240236282349
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,2,128,1,float16,fp8,0,0.10856479406356812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,2,128,1,float16,float16,0,0.11726399660110473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,4,128,1,float16,float16,0,0.1181007981300354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,4,128,1,float16,fp8,0,0.1091312050819397
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,4,128,1,fp8,fp8,0,0.10971360206604004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,8,128,1,float16,float16,0,0.12142879962921142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,8,128,1,float16,fp8,0,0.11227200031280518
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,8,128,1,fp8,fp8,0,0.11345280408859253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,24,2,128,1,float16,float16,0,1.7176832199096679
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,24,24,128,1,float16,fp8,0,0.23191840648651124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,24,24,128,1,float16,float16,0,0.5122975826263427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,24,1,128,1,float16,fp8,0,3.08548641204834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,2,128,1,fp8,fp8,0,0.10891040563583373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,24,1,128,1,float16,float16,0,3.379374313354492
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,24,2,128,1,float16,float16,0,3.3840095520019533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,24,2,128,1,float16,fp8,0,3.132111930847168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,24,2,128,1,fp8,fp8,0,3.125587272644043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,24,4,128,1,float16,fp8,0,3.1348592758178713
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,24,24,128,1,fp8,fp8,0,0.12247359752655029
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,24,4,128,1,fp8,fp8,0,3.2134624481201173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,24,1,128,1,fp8,fp8,0,3.1084192276000975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,24,8,128,1,float16,fp8,0,3.229075241088867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,24,8,128,1,float16,float16,0,3.656835174560547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,24,128,1,float16,fp8,0,1.7695791244506835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,1,128,1,float16,float16,0,1.609881591796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,24,128,1,float16,float16,0,2.010972785949707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,24,128,1,fp8,fp8,0,1.8006591796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,24,8,128,1,fp8,fp8,0,3.226675033569336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,1,128,1,float16,fp8,0,1.5542847633361816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,1,128,1,fp8,fp8,0,1.6556207656860351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,2,128,1,float16,float16,0,1.6075103759765625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,24,4,128,1,float16,float16,0,3.554270553588867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,4,128,1,float16,float16,0,1.6547760009765624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,2,128,1,fp8,fp8,0,1.6377103805541993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,4,128,1,float16,fp8,0,1.577012825012207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,4,128,1,fp8,fp8,0,1.5743904113769531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,8,128,1,float16,float16,0,1.7676031112670898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,8,128,1,float16,fp8,0,1.6758207321166991
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,24,128,1,fp8,fp8,0,0.8989551544189454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,1,128,1,float16,float16,0,0.8006912231445312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,1,128,1,float16,fp8,0,0.8675696372985839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,1,128,1,fp8,fp8,0,0.7855167865753174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,2,128,1,float16,float16,0,0.8072640419006347
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,2,128,1,float16,fp8,0,0.8423600196838379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,2,128,1,fp8,fp8,0,0.8133968353271485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,4,128,1,float16,float16,0,0.8290080070495606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,4,128,1,float16,fp8,0,0.795900821685791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,4,128,1,fp8,fp8,0,0.8193792343139649
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,8,128,1,float16,float16,0,0.8660976409912109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,8,128,1,float16,fp8,0,0.837553596496582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,24,128,1,float16,float16,0,0.5054687976837158
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,24,128,1,float16,fp8,0,0.9140607833862304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,8,128,1,fp8,fp8,0,0.8159664154052735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,24,128,1,float16,fp8,0,0.4743951797485352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,24,128,1,fp8,fp8,0,0.45882081985473633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,8,128,1,fp8,fp8,0,1.615065574645996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,1,128,1,float16,fp8,0,0.4003007888793945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,1,128,1,fp8,fp8,0,0.39710400104522703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,2,128,1,float16,float16,0,0.41269760131835936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,2,128,1,fp8,fp8,0,0.4009903907775879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,2,128,1,float16,fp8,0,0.3975888013839722
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,4,128,1,float16,float16,0,0.41760478019714353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,4,128,1,float16,fp8,0,0.4057168006896973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,4,128,1,fp8,fp8,0,0.4067359924316406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,8,128,1,float16,float16,0,0.4339183807373047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,8,128,1,float16,fp8,0,0.41376800537109376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,24,128,1,float16,float16,0,0.2607088088989258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,24,128,1,fp8,fp8,0,0.23554720878601074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,24,2,128,1,float16,fp8,0,1.5788463592529296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,1,128,1,float16,fp8,0,0.2046816110610962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,1,128,1,fp8,fp8,0,0.20594398975372313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,2,128,1,float16,fp8,0,0.20638399124145507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,2,128,1,fp8,fp8,0,0.20643200874328613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,24,24,128,1,float16,float16,0,1.0016240119934081
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,4,128,1,float16,float16,0,0.21856958866119386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,1,128,1,float16,float16,0,0.4112383842468262
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,4,128,1,fp8,fp8,0,0.2099087953567505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,8,128,1,float16,float16,0,0.22468318939208984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,8,128,1,float16,fp8,0,0.2146496057510376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,8,128,1,fp8,fp8,0,0.21419839859008788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,24,128,1,float16,fp8,0,0.12486560344696045
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,24,128,1,fp8,fp8,0,0.1251423954963684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,1,128,1,float16,float16,0,0.1113055944442749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,24,8,128,1,fp8,fp8,0,0.4150383949279785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,1,128,1,fp8,fp8,0,0.10584640502929688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,1,128,1,float16,float16,0,0.2105168104171753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,2,128,1,float16,float16,0,0.11256159543991089
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,2,128,1,float16,fp8,0,0.10670720338821411
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,2,128,1,fp8,fp8,0,0.10638079643249512
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,4,128,1,float16,float16,0,0.1146880030632019
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,4,128,1,float16,fp8,0,0.1090880036354065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,4,128,1,fp8,fp8,0,0.10930240154266357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,8,128,1,float16,float16,0,0.1195296049118042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,8,128,1,float16,fp8,0,0.11236640214920043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,8,128,1,fp8,fp8,0,0.11281759738922119
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,24,128,1,float16,float16,0,0.08129760026931762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,24,128,1,float16,fp8,0,0.07206720113754272
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,24,128,1,fp8,fp8,0,0.07222560048103333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,1,128,1,float16,float16,0,0.06044639945030213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,1,128,1,fp8,fp8,0,0.05850719809532166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,2,128,1,float16,float16,0,0.060755199193954466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,4,128,1,float16,fp8,0,0.211081600189209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,2,128,1,float16,fp8,0,0.05857920050621033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,2,128,1,fp8,fp8,0,0.059062397480010985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,4,128,1,float16,float16,0,0.06153920292854309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,4,128,1,fp8,fp8,0,0.05955680012702942
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,8,128,1,float16,float16,0,0.06449120044708252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,8,128,1,float16,fp8,0,0.06023039817810059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,8,128,1,fp8,fp8,0,0.060099202394485476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,24,128,1,float16,float16,0,0.13887840509414673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,24,128,1,float16,fp8,0,0.2355679988861084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,24,2,128,1,float16,float16,0,0.21150720119476318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,24,1,128,1,float16,float16,0,1.9955120086669922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,24,1,128,1,fp8,fp8,0,1.892136001586914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,1,128,1,float16,fp8,0,0.0588815987110138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,24,2,128,1,float16,float16,0,1.936476707458496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,24,4,128,1,float16,fp8,0,0.058911997079849246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,24,2,128,1,float16,fp8,0,1.9128623962402345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,24,2,128,1,fp8,fp8,0,2.0284191131591798
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,24,4,128,1,float16,float16,0,2.077057647705078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,24,1,128,1,float16,fp8,0,0.10626399517059326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,24,4,128,1,float16,fp8,0,1.9533920288085938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,24,4,128,1,fp8,fp8,0,2.0454864501953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,24,8,128,1,float16,float16,0,2.133492851257324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,24,1,128,1,float16,fp8,0,1.9214975357055664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,24,128,1,float16,float16,0,1.3309359550476074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,24,128,1,float16,fp8,0,1.1450608253479004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,24,8,128,1,float16,fp8,0,2.0066207885742187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,24,128,1,fp8,fp8,0,1.142255973815918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,24,8,128,1,fp8,fp8,0,2.0102560043334963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,1,128,1,float16,float16,0,0.973532772064209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,1,128,1,fp8,fp8,0,0.9599696159362793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,2,128,1,float16,fp8,0,0.9678607940673828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,2,128,1,fp8,fp8,0,0.9613200187683105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,4,128,1,float16,float16,0,1.0103327751159668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,4,128,1,float16,fp8,0,0.9853648185729981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,4,128,1,fp8,fp8,0,1.0648768424987793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,8,128,1,float16,float16,0,1.0483663558959961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,24,128,1,float16,float16,0,0.6556320190429688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,8,128,1,fp8,fp8,0,1.0119711875915527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,24,128,1,fp8,fp8,0,0.5892015933990479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,1,128,1,float16,float16,0,0.5035232067108154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,1,128,1,float16,fp8,0,0.507204818725586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,1,128,1,fp8,fp8,0,0.48636641502380373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,2,128,1,float16,float16,0,0.5020336151123047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,2,128,1,float16,fp8,0,0.49475998878479005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,2,128,1,fp8,fp8,0,0.49837121963500974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,4,128,1,float16,float16,0,0.5094207763671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,4,128,1,fp8,fp8,0,0.5040207862854004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,8,128,1,float16,float16,0,0.5387936115264893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,8,128,1,float16,fp8,0,0.5129295825958252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,8,128,1,fp8,fp8,0,0.5170703887939453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,2,128,1,float16,float16,0,1.0219440460205078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,24,128,1,float16,float16,0,0.3256911993026733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,24,128,1,float16,fp8,0,0.2982208013534546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,24,128,1,fp8,fp8,0,0.30057759284973146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,1,128,1,float16,float16,0,0.25679199695587157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,1,128,1,float16,fp8,0,0.24891679286956786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,1,128,1,fp8,fp8,0,0.2516848087310791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,2,128,1,float16,float16,0,0.25839359760284425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,2,128,1,float16,fp8,0,0.2496959924697876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,2,128,1,fp8,fp8,0,0.25190560817718505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,8,128,1,float16,fp8,0,1.0119104385375977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,4,128,1,float16,float16,0,0.26507999897003176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,24,128,1,float16,fp8,0,0.5816815853118896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,4,128,1,fp8,fp8,0,0.2567008018493652
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,8,128,1,float16,float16,0,0.2740400075912476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,8,128,1,fp8,fp8,0,0.26469600200653076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,24,128,1,float16,float16,0,0.17118240594863893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,24,128,1,float16,fp8,0,0.15941280126571655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,24,128,1,fp8,fp8,0,0.15981919765472413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,1,128,1,float16,fp8,0,0.13191200494766236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,1,128,1,fp8,fp8,0,0.13101119995117189
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,2,128,1,float16,float16,0,0.13563040494918824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,2,128,1,float16,fp8,0,0.13173120021820067
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,24,4,128,1,float16,fp8,0,0.5010159969329834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,2,128,1,fp8,fp8,0,0.13271039724349976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,4,128,1,float16,float16,0,0.13842079639434815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,4,128,1,float16,fp8,0,0.13337119817733764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,4,128,1,fp8,fp8,0,0.13409440517425536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,8,128,1,float16,float16,0,0.1444640040397644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,8,128,1,float16,fp8,0,0.13798880577087402
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,8,128,1,fp8,fp8,0,0.13883839845657348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,24,1,128,1,float16,fp8,0,1.039350414276123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,24,128,1,float16,fp8,0,0.08918079733848572
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,24,128,1,fp8,fp8,0,0.08956800103187561
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,1,128,1,float16,float16,0,0.07072319984436035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,1,128,1,fp8,fp8,0,0.06848480105400086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,2,128,1,float16,float16,0,0.07284640073776245
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,2,128,1,float16,fp8,0,0.06842880249023438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,2,128,1,fp8,fp8,0,0.06894400119781494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,4,128,1,float16,float16,0,0.07565919756889343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,4,128,1,float16,fp8,0,0.06999679803848266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,4,128,1,fp8,fp8,0,0.07037119865417481
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,8,128,1,float16,float16,0,0.07927359938621521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,8,128,1,float16,fp8,0,0.07362719774246215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,8,128,1,fp8,fp8,0,0.07356320023536682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,24,128,1,float16,float16,0,0.05722399950027466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,24,128,1,float16,fp8,0,0.0498879998922348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,24,128,1,fp8,fp8,0,0.04981119930744171
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,1,128,1,float16,float16,0,0.04124319851398468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,1,128,1,float16,fp8,0,0.040299201011657716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,1,128,1,fp8,fp8,0,0.04061920046806335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,2,128,1,float16,float16,0,0.041566398739814756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,2,128,1,float16,fp8,0,0.04058879911899567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,4,128,1,float16,float16,0,0.041998401284217834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,4,128,1,float16,fp8,0,0.04096480011940003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,8,128,1,float16,fp8,0,0.2657344102859497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,8,128,1,float16,float16,0,0.04357919991016388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,8,128,1,float16,fp8,0,0.04143039882183075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,8,128,1,fp8,fp8,0,0.041499200463294986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,24,1,128,1,float16,float16,0,0.1339792013168335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,24,128,1,float16,float16,0,0.09167519807815552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,24,1,128,1,float16,fp8,0,0.06833440065383911
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,24,1,128,1,float16,float16,0,2.0095455169677736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,24,1,128,1,float16,fp8,0,2.0205984115600586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,24,1,128,1,fp8,fp8,0,2.0171440124511717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,24,4,128,1,float16,fp8,0,0.2542351961135864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,2,128,1,fp8,fp8,0,0.040905600786209105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,24,4,128,1,fp8,fp8,0,0.040556800365447995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,24,2,128,1,float16,float16,0,2.0457584381103517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,24,2,128,1,float16,fp8,0,2.005534362792969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,24,2,128,1,fp8,fp8,0,2.067132759094238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,24,4,128,1,float16,float16,0,2.118351936340332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,24,4,128,1,float16,fp8,0,2.048062324523926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,24,4,128,1,fp8,fp8,0,2.0495088577270506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,24,128,1,float16,float16,0,1.3808544158935547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,24,8,128,1,float16,float16,0,2.2264192581176756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,24,8,128,1,float16,fp8,0,2.1572160720825195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,24,128,1,float16,fp8,0,1.2461119651794434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,1,128,1,float16,float16,0,1.0016143798828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,24,128,1,fp8,fp8,0,1.2779088020324707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,24,8,128,1,fp8,fp8,0,2.2504095077514648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,1,128,1,float16,fp8,0,1.006833553314209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,1,128,1,fp8,fp8,0,1.0596559524536133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,2,128,1,fp8,fp8,0,1.0097295761108398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,4,128,1,float16,float16,0,1.046224021911621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,4,128,1,float16,fp8,0,1.0456624031066895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,4,128,1,fp8,fp8,0,1.0326144218444824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,8,128,1,float16,float16,0,1.1091343879699707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,24,128,1,float16,float16,0,0.7117536067962646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,24,128,1,float16,fp8,0,0.6359344005584717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,24,128,1,fp8,fp8,0,0.6338304042816162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,8,128,1,fp8,fp8,0,1.0788175582885742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,1,128,1,float16,float16,0,0.5090928077697754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,1,128,1,float16,fp8,0,0.5165279865264892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,1,128,1,fp8,fp8,0,0.5143424034118652
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,2,128,1,float16,float16,0,0.518225622177124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,2,128,1,float16,fp8,0,0.512886381149292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,2,128,1,fp8,fp8,0,0.5109119892120362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,4,128,1,float16,float16,0,0.5372896194458008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,4,128,1,float16,fp8,0,0.5272047996520997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,4,128,1,fp8,fp8,0,0.5253280162811279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,8,128,1,float16,float16,0,0.5652688026428223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,24,128,1,float16,float16,0,0.3623631954193115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,8,128,1,fp8,fp8,0,0.5510479927062988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,24,128,1,fp8,fp8,0,0.327673602104187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,1,128,1,float16,float16,0,0.26268959045410156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,2,128,1,float16,fp8,0,1.010814380645752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,1,128,1,float16,fp8,0,0.2640239953994751
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,2,128,1,float16,float16,0,0.26488161087036133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,2,128,1,float16,fp8,0,0.2648688077926636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,2,128,1,fp8,fp8,0,0.263319993019104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,4,128,1,float16,float16,0,0.27513599395751953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,8,128,1,float16,fp8,0,1.0925951957702638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,4,128,1,float16,fp8,0,0.27005600929260254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,4,128,1,fp8,fp8,0,0.27041599750518797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,8,128,1,float16,float16,0,0.2904639959335327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,8,128,1,float16,fp8,0,0.2816096067428589
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,24,128,1,float16,float16,0,0.19064639806747435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,24,128,1,float16,fp8,0,0.17220159769058227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,24,128,1,fp8,fp8,0,0.17207839488983154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,1,128,1,float16,float16,0,0.13901760578155517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,1,128,1,float16,fp8,0,0.13840160369873047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,1,128,1,fp8,fp8,0,0.13855199813842772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,2,128,1,float16,float16,0,0.14029760360717775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,2,128,1,float16,fp8,0,0.13830560445785522
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,2,128,1,fp8,fp8,0,0.13799840211868286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,24,128,1,float16,fp8,0,0.3262079954147339
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,4,128,1,float16,float16,0,0.14528160095214843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,4,128,1,float16,fp8,0,0.14116959571838378
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,4,128,1,fp8,fp8,0,0.14187040328979492
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,8,128,1,float16,float16,0,0.1532304048538208
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,8,128,1,float16,fp8,0,0.14762879610061647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,24,8,128,1,fp8,fp8,0,0.1479024052619934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,24,128,1,float16,float16,0,0.10371359586715698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,24,128,1,float16,fp8,0,0.09443680047988892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,1,128,1,fp8,fp8,0,0.2635616064071655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,1,128,1,float16,fp8,0,0.07272480130195617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,1,128,1,fp8,fp8,0,0.0739408016204834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,2,128,1,float16,float16,0,0.07740319967269897
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,2,128,1,float16,fp8,0,0.074508798122406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,2,128,1,fp8,fp8,0,0.07440000176429748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,4,128,1,float16,float16,0,0.08025280237197877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,4,128,1,fp8,fp8,0,0.07720320224761963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,8,128,1,float16,float16,0,0.0842848002910614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,8,128,1,float16,fp8,0,0.08059359788894653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,8,128,1,fp8,fp8,0,0.08059999942779542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,24,128,1,float16,float16,0,0.06078879833221436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,24,8,128,1,fp8,fp8,0,0.28389439582824705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,24,128,1,float16,fp8,0,0.05664479732513428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,24,128,1,fp8,fp8,0,0.055576002597808837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,1,128,1,float16,fp8,0,0.04068480134010315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,1,128,1,fp8,fp8,0,0.0406607985496521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,2,128,1,float16,float16,0,0.040227198600769044
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,2,128,1,fp8,fp8,0,0.04077439904212952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,2,128,1,float16,fp8,0,0.040356799960136414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,4,128,1,float16,float16,0,0.04106720089912415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,4,128,1,float16,fp8,0,0.04164800047874451
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,4,128,1,fp8,fp8,0,0.04222399890422821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,8,128,1,float16,float16,0,0.0449535995721817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,8,128,1,float16,fp8,0,0.04321599900722504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,8,128,1,fp8,fp8,0,0.04276480078697205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,24,128,1,float16,float16,0,0.03320479989051819
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,24,128,1,float16,fp8,0,0.031167998909950256
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,24,2,128,1,float16,float16,0,1.0341423988342284
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,1,128,1,float16,float16,0,0.024350400269031524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,1,128,1,fp8,fp8,0,0.02484000027179718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,1,128,1,float16,fp8,0,0.02484000027179718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,2,128,1,float16,float16,0,0.02446240037679672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,24,128,1,fp8,fp8,0,0.09494559764862061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,2,128,1,fp8,fp8,0,0.024916799366474153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,4,128,1,float16,float16,0,0.024592000246047973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,4,128,1,float16,fp8,0,0.02611519992351532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,4,128,1,fp8,fp8,0,0.026240000128746034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,8,128,1,float16,fp8,0,0.026715201139450074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,4,128,1,float16,fp8,0,0.07623839974403382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,24,1,128,1,float16,float16,0,0.040033599734306334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,24,1,128,1,float16,float16,0,1.390073585510254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,24,8,128,1,float16,fp8,0,0.5462224006652832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,24,128,1,fp8,fp8,0,0.031523200869560244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,24,1,128,1,float16,fp8,0,1.4457488059997559
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,2,128,1,float16,fp8,0,0.024723200500011443
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,24,1,128,1,fp8,fp8,0,1.438081645965576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,8,128,1,float16,float16,0,0.025339201092720032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,24,2,128,1,float16,float16,0,1.4207216262817384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,24,8,128,1,fp8,fp8,0,0.0265392005443573
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,24,2,128,1,float16,fp8,0,1.4491488456726074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,24,2,128,1,fp8,fp8,0,1.4464768409729003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,24,4,128,1,float16,float16,0,1.4890624046325684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,24,4,128,1,float16,fp8,0,1.4827280044555664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,24,4,128,1,fp8,fp8,0,1.4856575965881347
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,24,8,128,1,float16,float16,0,1.619772720336914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,24,8,128,1,float16,fp8,0,1.584609603881836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,24,128,1,float16,float16,0,1.0860735893249511
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,24,128,1,float16,fp8,0,0.9924847602844238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,1,128,1,float16,float16,0,0.7051504135131836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,24,8,128,1,fp8,fp8,0,1.5923295974731446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,24,128,1,fp8,fp8,0,0.9860383987426757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,1,128,1,float16,fp8,0,0.7475008010864258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,1,128,1,fp8,fp8,0,0.7272175788879395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,24,1,128,1,float16,float16,0,0.07644960284233093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,2,128,1,float16,float16,0,0.7180079936981201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,2,128,1,float16,fp8,0,0.7312352180480957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,2,128,1,fp8,fp8,0,0.7275343894958496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,4,128,1,float16,float16,0,0.7499472141265869
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,4,128,1,fp8,fp8,0,0.7656400203704834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,8,128,1,float16,float16,0,0.8099007606506348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,8,128,1,float16,fp8,0,0.807307243347168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,24,128,1,float16,float16,0,0.5519631862640381
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,8,128,1,fp8,fp8,0,0.8025152206420898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,1,128,1,float16,float16,0,0.35819039344787595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,24,128,1,fp8,fp8,0,0.5072912216186524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,1,128,1,float16,fp8,0,0.37374880313873293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,1,128,1,fp8,fp8,0,0.3696127891540527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,2,128,1,float16,float16,0,0.3651792049407959
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,2,128,1,float16,fp8,0,0.3695055961608887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,2,128,1,fp8,fp8,0,0.3752959966659546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,4,128,1,float16,float16,0,0.3834991931915283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,4,128,1,fp8,fp8,0,0.3835968017578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,8,128,1,float16,float16,0,0.4122352123260498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,8,128,1,float16,fp8,0,0.41183037757873536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,24,128,1,float16,float16,0,0.2831824064254761
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,24,128,1,float16,fp8,0,0.25813438892364504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,1,128,1,float16,float16,0,0.18628000020980834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,1,128,1,float16,fp8,0,0.19112160205841064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,1,128,1,fp8,fp8,0,0.1914080023765564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,2,128,1,float16,float16,0,0.18909759521484376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,2,128,1,float16,fp8,0,0.19175679683685304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,2,128,1,fp8,fp8,0,0.1910591959953308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,4,128,1,float16,float16,0,0.19732320308685303
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,24,4,128,1,float16,fp8,0,0.74934401512146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,4,128,1,float16,fp8,0,0.197542405128479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,4,128,1,fp8,fp8,0,0.19834879636764527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,8,128,1,float16,float16,0,0.21169600486755372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,24,128,1,float16,float16,0,0.14855040311813356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,24,128,1,float16,fp8,0,0.13662079572677613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,24,128,1,fp8,fp8,0,0.1366703987121582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,1,128,1,float16,float16,0,0.09919520020484925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,4,128,1,float16,fp8,0,0.38170719146728516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,1,128,1,fp8,fp8,0,0.10141600370407104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,2,128,1,float16,float16,0,0.10099040269851685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,2,128,1,float16,fp8,0,0.10175199508666992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,24,128,1,fp8,fp8,0,0.2613248109817505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,4,128,1,float16,float16,0,0.10561599731445312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,4,128,1,float16,fp8,0,0.10633599758148193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,4,128,1,fp8,fp8,0,0.10511519908905029
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,8,128,1,float16,float16,0,0.11483039855957031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,8,128,1,float16,fp8,0,0.11203199625015259
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,8,128,1,fp8,fp8,0,0.1132464051246643
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,24,128,1,float16,float16,0,0.08254560232162475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,24,128,1,fp8,fp8,0,0.07626240253448487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,1,128,1,float16,float16,0,0.056036800146102905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,8,128,1,float16,fp8,0,0.20996479988098143
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,1,128,1,fp8,fp8,0,0.05421440005302429
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,24,128,1,float16,fp8,0,0.5035935878753662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,2,128,1,float16,float16,0,0.05668799877166748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,2,128,1,float16,fp8,0,0.055011200904846194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,4,128,1,float16,float16,0,0.059564799070358276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,4,128,1,float16,fp8,0,0.05772960186004639
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,4,128,1,fp8,fp8,0,0.05772960186004639
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,1,128,1,float16,fp8,0,0.10132160186767578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,8,128,1,float16,fp8,0,0.06245120167732239
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,8,128,1,fp8,fp8,0,0.0622767984867096
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,24,8,128,1,fp8,fp8,0,0.41105117797851565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,24,128,1,float16,float16,0,0.0478879988193512
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,24,128,1,fp8,fp8,0,0.04406880140304566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,1,128,1,float16,float16,0,0.03155680000782013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,1,128,1,float16,fp8,0,0.031620800495147705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,1,128,1,fp8,fp8,0,0.03165760040283203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,2,128,1,float16,float16,0,0.031856000423431396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,2,128,1,float16,fp8,0,0.031676799058914185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,2,128,1,fp8,fp8,0,0.031302401423454286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,4,128,1,float16,float16,0,0.032630398869514465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,24,128,1,float16,fp8,0,0.07697920203208923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,4,128,1,float16,fp8,0,0.03240480124950409
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,4,128,1,fp8,fp8,0,0.03244960010051727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,8,128,1,float16,float16,0,0.035657599568367004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,8,128,1,float16,fp8,0,0.034067198634147644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,8,128,1,fp8,fp8,0,0.033899199962615964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,24,128,1,float16,float16,0,0.02844479978084564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,24,128,1,float16,fp8,0,0.028244799375534056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,1,128,1,float16,fp8,0,0.05469759702682495
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,1,128,1,float16,float16,0,0.020678399503231047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,1,128,1,float16,fp8,0,0.020283199846744537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,1,128,1,fp8,fp8,0,0.020478400588035583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,2,128,1,float16,float16,0,0.020820799469947814
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,2,128,1,float16,fp8,0,0.020577600598335265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,2,128,1,fp8,fp8,0,0.02036159932613373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,4,128,1,float16,float16,0,0.021187199652194975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,4,128,1,float16,fp8,0,0.022566400468349457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,2,128,1,fp8,fp8,0,0.05533599853515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,4,128,1,fp8,fp8,0,0.022324800491333008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,8,128,1,float16,float16,0,0.022012799978256226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,8,128,1,float16,fp8,0,0.022835199534893037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,24,128,1,float16,float16,0,0.019280000030994414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,24,128,1,float16,fp8,0,0.02000479996204376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,24,128,1,fp8,fp8,0,0.01995519995689392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,1,128,1,float16,float16,0,0.017459200322628023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,1,128,1,float16,fp8,0,0.01863040030002594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,1,128,1,fp8,fp8,0,0.01866399943828583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,2,128,1,float16,float16,0,0.017505599558353423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,2,128,1,float16,fp8,0,0.018739199638366698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,2,128,1,fp8,fp8,0,0.018619200587272643
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,4,128,1,float16,float16,0,0.017576000094413756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,4,128,1,float16,fp8,0,0.019057600200176238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,4,128,1,fp8,fp8,0,0.019204799830913544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,8,128,1,float16,float16,0,0.018199999630451203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,8,128,1,float16,fp8,0,0.01937599927186966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,24,24,128,1,float16,fp8,0,0.04438399970531463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,24,8,128,1,fp8,fp8,0,0.019436800479888917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,24,2,128,1,fp8,fp8,0,0.10160479545593262
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,24,1,128,1,float16,float16,0,0.567903995513916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,24,1,128,1,float16,fp8,0,0.5956064224243164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,24,1,128,1,fp8,fp8,0,0.5991968154907227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,24,2,128,1,float16,float16,0,0.5793216228485107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,24,2,128,1,float16,fp8,0,0.5960512161254883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,24,128,1,fp8,fp8,0,0.028352001309394838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,24,2,128,1,fp8,fp8,0,0.6009903907775879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,24,8,128,1,fp8,fp8,0,0.21044158935546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,24,4,128,1,float16,float16,0,0.6119328022003174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,24,4,128,1,float16,fp8,0,0.6202527999877929
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,24,4,128,1,fp8,fp8,0,0.6254127979278564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,24,8,128,1,float16,float16,0,0.6723455905914306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,24,8,128,1,float16,fp8,0,0.6733183860778809
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,24,8,128,1,float16,float16,0,0.06423519849777222
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,24,128,1,float16,float16,0,0.48053441047668455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,24,8,128,1,fp8,fp8,0,0.6772736072540283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,1,128,1,float16,fp8,0,0.3063024044036865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,1,128,1,fp8,fp8,0,0.30385119915008546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,2,128,1,float16,float16,0,0.29687039852142333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,2,128,1,float16,fp8,0,0.3069904088973999
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,2,128,1,fp8,fp8,0,0.30485119819641116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,4,128,1,float16,float16,0,0.3128175973892212
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,4,128,1,float16,fp8,0,0.3197135925292969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,4,128,1,fp8,fp8,0,0.3189743995666504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,8,128,1,float16,float16,0,0.34282560348510743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,8,128,1,float16,fp8,0,0.34582879543304446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,8,128,1,fp8,fp8,0,0.34414401054382326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,24,128,1,float16,fp8,0,0.21541600227355956
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,1,128,1,float16,float16,0,0.15218240022659302
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,24,128,1,fp8,fp8,0,0.2167135953903198
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,1,128,1,float16,fp8,0,0.1595039963722229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,1,128,1,fp8,fp8,0,0.15983999967575074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,2,128,1,float16,float16,0,0.15600960254669188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,2,128,1,float16,fp8,0,0.15993280410766603
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,2,128,1,fp8,fp8,0,0.16060800552368165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,4,128,1,float16,float16,0,0.163428795337677
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,4,128,1,float16,fp8,0,0.16654880046844484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,24,8,128,1,fp8,fp8,0,0.022697600722312927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,4,128,1,fp8,fp8,0,0.16737760305404664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,8,128,1,float16,float16,0,0.17984639406204223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,8,128,1,float16,fp8,0,0.17872159481048583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,8,128,1,fp8,fp8,0,0.17845280170440675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,24,128,1,float16,float16,0,0.13253920078277587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,1,128,1,float16,float16,0,0.08156639933586121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,1,128,1,float16,fp8,0,0.08670560121536255
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,1,128,1,fp8,fp8,0,0.08608959913253784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,24,128,1,float16,fp8,0,0.41775999069213865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,24,128,1,fp8,fp8,0,0.41648478507995607
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,2,128,1,float16,fp8,0,0.08629440069198609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,2,128,1,fp8,fp8,0,0.08643680214881896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,4,128,1,float16,float16,0,0.08825119733810424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,4,128,1,fp8,fp8,0,0.08951039910316468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,8,128,1,float16,float16,0,0.09643359780311585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,8,128,1,float16,fp8,0,0.09621760249137878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,8,128,1,fp8,fp8,0,0.09572799801826477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,24,128,1,float16,float16,0,0.07376639842987061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,24,128,1,float16,fp8,0,0.06422079801559448
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,24,128,1,fp8,fp8,0,0.06415200233459473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,1,128,1,float16,float16,0,0.04752320051193237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,1,128,1,float16,fp8,0,0.04620000123977661
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,1,128,1,fp8,fp8,0,0.046419200301170346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,2,128,1,float16,float16,0,0.048819199204444885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,2,128,1,float16,fp8,0,0.04741120040416717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,2,128,1,fp8,fp8,0,0.047881600260734555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,4,128,1,float16,float16,0,0.051369601488113405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,24,24,128,1,float16,float16,0,0.24921119213104248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,4,128,1,float16,fp8,0,0.04995200037956238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,4,128,1,fp8,fp8,0,0.04963839948177338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,8,128,1,float16,float16,0,0.055852800607681274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,8,128,1,float16,fp8,0,0.05425440073013306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,24,128,1,float16,float16,0,0.04210399985313416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,24,128,1,float16,fp8,0,0.03735840022563934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,24,128,1,fp8,fp8,0,0.036924800276756285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,1,128,1,float16,float16,0,0.026897600293159483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,1,128,1,float16,fp8,0,0.027886399626731874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,1,128,1,fp8,fp8,0,0.02771199941635132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,2,128,1,float16,float16,0,0.027195200324058533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,2,128,1,float16,fp8,0,0.027774399518966673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,2,128,1,fp8,fp8,0,0.027715200185775758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,4,128,1,float16,float16,0,0.027995198965072632
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,4,128,1,float16,fp8,0,0.028123199939727783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,4,128,1,fp8,fp8,0,0.028297600150108338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,8,128,1,float16,float16,0,0.030972799658775328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,8,128,1,float16,fp8,0,0.030820798873901368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,24,8,128,1,fp8,fp8,0,0.03017919957637787
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,24,128,1,float16,float16,0,0.02473440021276474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,24,128,1,float16,fp8,0,0.022409600019454957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,24,128,1,fp8,fp8,0,0.02223840057849884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,1,128,1,float16,float16,0,0.018855999410152435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,1,128,1,float16,fp8,0,0.020287999510765077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,2,128,1,float16,float16,0,0.01891999989748001
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,2,128,1,float16,fp8,0,0.020347200334072113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,24,1,128,1,float16,float16,0,0.2895472049713135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,2,128,1,fp8,fp8,0,0.02024320065975189
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,4,128,1,float16,fp8,0,0.0893343985080719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,4,128,1,float16,float16,0,0.019310399889945984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,4,128,1,float16,fp8,0,0.020740799605846405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,8,128,1,float16,float16,0,0.02006080001592636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,8,128,1,float16,fp8,0,0.02106879949569702
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,8,128,1,fp8,fp8,0,0.0212351992726326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,24,128,1,float16,float16,0,0.016364799439907075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,24,128,1,float16,fp8,0,0.01650879979133606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,24,128,1,fp8,fp8,0,0.016542400419712066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,1,128,1,float16,float16,0,0.014609600603580474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,1,128,1,float16,fp8,0,0.01518079936504364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,1,128,1,fp8,fp8,0,0.015134400129318238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,2,128,1,float16,float16,0,0.014636799693107605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,24,8,128,1,fp8,fp8,0,0.05417760014533997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,2,128,1,float16,fp8,0,0.015142400562763215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,2,128,1,fp8,fp8,0,0.01526239961385727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,4,128,1,float16,fp8,0,0.015574400126934052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,4,128,1,fp8,fp8,0,0.015422399342060088
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,8,128,1,float16,float16,0,0.015185600519180298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,8,128,1,float16,fp8,0,0.015663999319076537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,8,128,1,fp8,fp8,0,0.01568640023469925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,24,128,1,float16,float16,0,0.015390400588512421
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,24,128,1,float16,fp8,0,0.015863999724388123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,24,128,1,fp8,fp8,0,0.015836800634860992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,24,128,1,float16,fp8,0,0.11573439836502075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,1,128,1,float16,float16,0,0.014305600523948669
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,1,128,1,fp8,fp8,0,0.020371200144290925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,1,128,1,fp8,fp8,0,0.015015999972820281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,2,128,1,float16,float16,0,0.014312000572681427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,2,128,1,float16,fp8,0,0.014982399344444276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,2,128,1,float16,float16,0,0.0845359981060028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,4,128,1,float16,float16,0,0.014446400105953217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,4,128,1,float16,fp8,0,0.015219199657440185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,4,128,1,fp8,fp8,0,0.015462400019168853
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,8,128,1,float16,float16,0,0.01462240070104599
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,8,128,1,float16,fp8,0,0.015289600193500518
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,8,128,1,fp8,fp8,0,0.015268799662590028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,24,4,128,1,fp8,fp8,0,0.020571200549602507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,24,1,128,1,float16,float16,0,0.27200319766998293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,24,1,128,1,float16,fp8,0,0.292902398109436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,24,1,128,1,fp8,fp8,0,0.2922111988067627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,24,2,128,1,float16,float16,0,0.27643680572509766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,24,2,128,1,float16,fp8,0,0.2965087890625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,24,2,128,1,fp8,fp8,0,0.29568800926208494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,24,4,128,1,float16,float16,0,0.2910799980163574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,24,4,128,1,float16,fp8,0,0.30691359043121336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,24,4,128,1,fp8,fp8,0,0.30619359016418457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,24,4,128,1,float16,float16,0,0.014851200580596923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,24,8,128,1,float16,fp8,0,0.33207039833068847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,24,8,128,1,fp8,fp8,0,0.3315887928009033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,24,128,1,float16,float16,0,0.23788158893585204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,24,128,1,float16,fp8,0,0.21615839004516602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,1,128,1,float16,float16,0,0.14148800373077391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,1,128,1,float16,fp8,0,0.15397919416427613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,1,128,1,fp8,fp8,0,0.14924800395965576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,2,128,1,float16,float16,0,0.14537279605865477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,1,128,1,float16,fp8,0,0.015121600031852723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,2,128,1,float16,fp8,0,0.15154880285263062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,24,2,128,1,fp8,fp8,0,0.014793600142002105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,2,128,1,fp8,fp8,0,0.15559359788894653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,4,128,1,float16,fp8,0,0.15851999521255494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,4,128,1,fp8,fp8,0,0.1608896017074585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,8,128,1,float16,float16,0,0.16757760047912598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,8,128,1,float16,fp8,0,0.17499519586563111
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,8,128,1,fp8,fp8,0,0.17304160594940185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,24,128,1,float16,fp8,0,0.1126255989074707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,24,128,1,fp8,fp8,0,0.11361440420150756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,1,128,1,float16,fp8,0,0.08268160223960877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,1,128,1,fp8,fp8,0,0.08279039859771728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,2,128,1,float16,float16,0,0.081004798412323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,2,128,1,float16,fp8,0,0.08299840092658997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,2,128,1,fp8,fp8,0,0.0827567994594574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,4,128,1,float16,float16,0,0.08497599959373474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,4,128,1,float16,fp8,0,0.08496479988098145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,8,128,1,float16,float16,0,0.09255200028419494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,8,128,1,float16,fp8,0,0.09100160002708435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,24,8,128,1,float16,float16,0,0.3206415891647339
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,24,128,1,float16,float16,0,0.07156640291213989
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,24,128,1,float16,fp8,0,0.06077759861946106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,24,128,1,fp8,fp8,0,0.06052640080451965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,24,128,1,fp8,fp8,0,0.21566081047058105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,1,128,1,float16,float16,0,0.04470719993114471
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,1,128,1,float16,fp8,0,0.04254719913005829
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,2,128,1,float16,float16,0,0.04512799978256225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,24,24,128,1,fp8,fp8,0,0.11475679874420167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,2,128,1,float16,fp8,0,0.04296799898147583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,2,128,1,fp8,fp8,0,0.0432559996843338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,4,128,1,float16,fp8,0,0.04508480131626129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,24,4,128,1,float16,float16,0,0.15270880460739136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,4,128,1,fp8,fp8,0,0.04527679979801178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,8,128,1,float16,float16,0,0.05256320238113403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,24,128,1,float16,float16,0,0.12711199522018432
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,8,128,1,fp8,fp8,0,0.04929440021514893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,24,128,1,float16,float16,0,0.040043199062347413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,24,128,1,fp8,fp8,0,0.03127999901771546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,1,128,1,float16,float16,0,0.023003199696540834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,1,128,1,float16,fp8,0,0.024332800507545473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,1,128,1,fp8,fp8,0,0.02433439940214157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,4,128,1,fp8,fp8,0,0.08542879819869995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,2,128,1,float16,float16,0,0.02337439954280853
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,2,128,1,float16,fp8,0,0.024560000002384185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,2,128,1,fp8,fp8,0,0.024540799856185912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,4,128,1,float16,float16,0,0.024094399809837342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,8,128,1,fp8,fp8,0,0.09089279770851136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,4,128,1,float16,fp8,0,0.024612799286842346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,4,128,1,fp8,fp8,0,0.024480000138282776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,8,128,1,float16,float16,0,0.02645759880542755
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,8,128,1,float16,fp8,0,0.024959999322891235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,24,128,1,float16,float16,0,0.022169600427150726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,24,128,1,float16,fp8,0,0.020308800041675568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,24,128,1,fp8,fp8,0,0.020326399803161622
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,1,128,1,float16,float16,0,0.01758880019187927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,1,128,1,float16,fp8,0,0.01870879977941513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,1,128,1,fp8,fp8,0,0.018646399676799773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,2,128,1,float16,float16,0,0.017617599666118623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,4,128,1,float16,float16,0,0.04847840070724487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,2,128,1,fp8,fp8,0,0.01866399943828583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,4,128,1,float16,float16,0,0.017977599799633027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,4,128,1,float16,fp8,0,0.018831999599933626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,8,128,1,float16,fp8,0,0.04931359887123108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,4,128,1,fp8,fp8,0,0.018775999546051025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,8,128,1,float16,float16,0,0.01884160041809082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,24,1,128,1,float16,float16,0,0.07927039861679078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,8,128,1,float16,fp8,0,0.018881599605083465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,8,128,1,fp8,fp8,0,0.019233599305152893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,24,128,1,float16,fp8,0,0.014286400377750396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,24,128,1,fp8,fp8,0,0.014528000354766845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,1,128,1,float16,float16,0,0.012644800543785095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,1,128,1,float16,fp8,0,0.013592000305652618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,1,128,1,fp8,fp8,0,0.013374400138854981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,2,128,1,float16,float16,0,0.013051199913024902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,2,128,1,float16,fp8,0,0.013363200426101684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,2,128,1,fp8,fp8,0,0.013619199395179749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,4,128,1,float16,float16,0,0.013070400059223174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,4,128,1,float16,fp8,0,0.013551999628543854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,4,128,1,fp8,fp8,0,0.013684800267219544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,8,128,1,float16,float16,0,0.013252800703048706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,8,128,1,float16,fp8,0,0.013831999897956849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,8,128,1,fp8,fp8,0,0.013704000413417817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,24,128,1,float16,float16,0,0.013502399623394012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,24,128,1,float16,fp8,0,0.013648000359535218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,24,128,1,fp8,fp8,0,0.013598400354385375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,1,128,1,float16,float16,0,0.012508800625801087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,1,128,1,float16,fp8,0,0.013115200400352477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,1,128,1,fp8,fp8,0,0.013359999656677246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,2,128,1,float16,float16,0,0.012579199671745301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,2,128,1,float16,fp8,0,0.013223999738693237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,2,128,1,fp8,fp8,0,0.013124799728393555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,4,128,1,float16,float16,0,0.012680000066757202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,4,128,1,float16,fp8,0,0.013212800025939941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,4,128,1,fp8,fp8,0,0.013078400492668152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,8,128,1,fp8,fp8,0,0.025198400020599365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,8,128,1,fp8,fp8,0,0.013150399923324585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,24,128,1,float16,float16,0,0.013084800541400909
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,24,128,1,float16,fp8,0,0.013342399895191193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,24,128,1,fp8,fp8,0,0.013665600121021271
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,1,128,1,float16,float16,0,0.012409599870443344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,1,128,1,float16,fp8,0,0.013220800459384919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,1,128,1,fp8,fp8,0,0.013123199343681335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,2,128,1,float16,float16,0,0.012467200309038163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,24,1,128,1,fp8,fp8,0,0.04191839993000031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,2,128,1,fp8,fp8,0,0.01311040073633194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,4,128,1,float16,float16,0,0.012356799840927125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,4,128,1,fp8,fp8,0,0.013065600395202636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,8,128,1,float16,float16,0,0.012827199697494508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,24,2,128,1,float16,fp8,0,0.01863040030002594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,8,128,1,float16,fp8,0,0.013129599392414093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,8,128,1,fp8,fp8,0,0.012913599610328674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,24,1,128,1,float16,float16,0,0.14429759979248047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,24,1,128,1,float16,fp8,0,0.1534719944000244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,24,1,128,1,fp8,fp8,0,0.1539023995399475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,24,24,128,1,float16,fp8,0,0.030932798981666565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,24,2,128,1,float16,float16,0,0.14564319849014282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,24,24,128,1,float16,float16,0,0.014732800424098969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,24,2,128,1,float16,fp8,0,0.15536799430847167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,24,2,128,1,fp8,fp8,0,0.15467679500579834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,24,4,128,1,float16,fp8,0,0.15939040184020997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,24,4,128,1,fp8,fp8,0,0.16066880226135255
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,24,8,128,1,float16,float16,0,0.1957103967666626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,24,8,128,1,float16,fp8,0,0.20345120429992675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,24,8,128,1,fp8,fp8,0,0.20560638904571532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,24,128,1,float16,float16,0,0.1703968048095703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,24,128,1,float16,fp8,0,0.1652127981185913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,24,128,1,fp8,fp8,0,0.1649616003036499
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,1,128,1,float16,fp8,0,0.08252000212669372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,1,128,1,fp8,fp8,0,0.08261439800262452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,2,128,1,float16,float16,0,0.08081759810447693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,2,128,1,float16,fp8,0,0.08372319936752319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,8,128,1,float16,float16,0,0.01273919939994812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,2,128,1,fp8,fp8,0,0.08331840038299561
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,24,8,128,1,float16,fp8,0,0.013451200723648072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,4,128,1,float16,fp8,0,0.08511679768562316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,4,128,1,fp8,fp8,0,0.08544800281524659
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,8,128,1,float16,float16,0,0.10547679662704468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,8,128,1,float16,fp8,0,0.10743039846420288
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,24,128,1,float16,float16,0,0.09246559739112854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,2,128,1,float16,fp8,0,0.013184000551700593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,24,128,1,float16,fp8,0,0.08759199976921081
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,24,4,128,1,float16,fp8,0,0.013254399597644805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,1,128,1,float16,float16,0,0.04494400024414062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,1,128,1,float16,fp8,0,0.04227199852466583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,2,128,1,float16,float16,0,0.046084800362586976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,2,128,1,float16,fp8,0,0.042671999335289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,2,128,1,fp8,fp8,0,0.0427839994430542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,4,128,1,float16,float16,0,0.047788798809051514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,4,128,1,float16,fp8,0,0.045844799280166625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,4,128,1,fp8,fp8,0,0.04594239890575409
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,24,4,128,1,float16,float16,0,0.15365920066833497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,8,128,1,float16,float16,0,0.05968639850616455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,8,128,1,float16,fp8,0,0.058449602127075194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,8,128,1,fp8,fp8,0,0.05815359950065613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,24,128,1,float16,float16,0,0.051742398738861085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,24,128,1,float16,fp8,0,0.04546079933643341
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,24,128,1,fp8,fp8,0,0.045423999428749084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,1,128,1,float16,float16,0,0.023307199776172637
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,1,128,1,float16,fp8,0,0.024825599789619446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,1,128,1,fp8,fp8,0,0.024792000651359558
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,2,128,1,float16,float16,0,0.02346560060977936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,1,128,1,float16,float16,0,0.07942559719085693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,2,128,1,fp8,fp8,0,0.024646399915218352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,4,128,1,float16,float16,0,0.02393440008163452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,4,128,1,float16,fp8,0,0.024806399643421174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,4,128,1,fp8,fp8,0,0.024843199551105498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,8,128,1,float16,float16,0,0.03059520125389099
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,8,128,1,float16,fp8,0,0.029747200012207032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,4,128,1,float16,float16,0,0.08473920226097106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,24,128,1,float16,fp8,0,0.024478399753570558
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,24,128,1,fp8,fp8,0,0.024454399943351746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,24,8,128,1,fp8,fp8,0,0.10728960037231446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,1,128,1,float16,fp8,0,0.018542400002479552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,1,128,1,fp8,fp8,0,0.018580800294876097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,2,128,1,float16,float16,0,0.017547200620174407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,1,128,1,fp8,fp8,0,0.041982400417327884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,2,128,1,float16,fp8,0,0.01918880045413971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,2,128,1,fp8,fp8,0,0.018636800348758698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,4,128,1,float16,float16,0,0.017846399545669557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,4,128,1,float16,fp8,0,0.018768000602722167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,8,128,1,float16,float16,0,0.01801439970731735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,8,128,1,float16,fp8,0,0.018982400000095368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,8,128,1,fp8,fp8,0,0.018984000384807586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,24,128,1,float16,float16,0,0.018296000361442567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,24,128,1,float16,fp8,0,0.018503999710083006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,24,128,1,fp8,fp8,0,0.018555200099945067
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,1,128,1,float16,float16,0,0.01271200031042099
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,1,128,1,float16,fp8,0,0.013649600744247436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,1,128,1,fp8,fp8,0,0.013532799482345582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,2,128,1,float16,float16,0,0.013115200400352477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,2,128,1,float16,fp8,0,0.013537600636482239
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,2,128,1,fp8,fp8,0,0.013816000521183014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,2,128,1,float16,fp8,0,0.024857600033283234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,4,128,1,float16,float16,0,0.013260799646377563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,4,128,1,fp8,fp8,0,0.013619199395179749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,8,128,1,float16,float16,0,0.013281600177288055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,8,128,1,float16,fp8,0,0.013595199584960938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,24,8,128,1,fp8,fp8,0,0.029956799745559693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,8,128,1,fp8,fp8,0,0.01361439973115921
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,24,128,1,float16,float16,0,0.02535040080547333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,24,128,1,float16,float16,0,0.013214400410652161
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,1,128,1,float16,float16,0,0.012425599992275238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,1,128,1,float16,fp8,0,0.01318880021572113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,1,128,1,fp8,fp8,0,0.013150399923324585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,2,128,1,float16,float16,0,0.012593600153923034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,2,128,1,float16,fp8,0,0.013124799728393555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,2,128,1,fp8,fp8,0,0.01329759955406189
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,4,128,1,float16,float16,0,0.01265919953584671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,4,128,1,float16,fp8,0,0.013329599797725678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,4,128,1,fp8,fp8,0,0.013041600584983826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,8,128,1,float16,float16,0,0.012809599936008453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,8,128,1,float16,fp8,0,0.013084800541400909
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,8,128,1,fp8,fp8,0,0.013209599256515502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,24,128,1,float16,float16,0,0.012582400441169738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,24,128,1,float16,fp8,0,0.013307200372219085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,24,128,1,fp8,fp8,0,0.01316480040550232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,1,128,1,float16,float16,0,0.012401600182056428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,1,128,1,float16,fp8,0,0.012870399653911591
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,4,128,1,fp8,fp8,0,0.018727999925613404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,1,128,1,fp8,fp8,0,0.013007999956607818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,2,128,1,float16,float16,0,0.012417600303888322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,2,128,1,float16,fp8,0,0.012895999848842621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,4,128,1,float16,float16,0,0.0125231996178627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,4,128,1,float16,fp8,0,0.012947200238704682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,4,128,1,fp8,fp8,0,0.013476799428462981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,8,128,1,float16,float16,0,0.012299200147390365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,8,128,1,float16,fp8,0,0.013121600449085235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,8,128,1,fp8,fp8,0,0.01308320015668869
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,24,128,1,float16,float16,0,0.012600000202655792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,24,128,1,float16,fp8,0,0.013299199938774108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,24,128,1,fp8,fp8,0,0.013249599933624267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,1,128,1,float16,float16,0,0.012191999703645706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,1,128,1,float16,fp8,0,0.012651200592517852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,1,128,1,fp8,fp8,0,0.012833599746227265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,2,128,1,float16,float16,0,0.012070400267839431
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,2,128,1,float16,fp8,0,0.01279039978981018
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,2,128,1,fp8,fp8,0,0.012724800407886505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,4,128,1,float16,float16,0,0.012142399698495865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,4,128,1,float16,fp8,0,0.012727999687194824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,4,128,1,fp8,fp8,0,0.012798400223255157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,8,128,1,float16,float16,0,0.012104000151157378
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,8,128,1,float16,fp8,0,0.012753599882125854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,24,8,128,1,fp8,fp8,0,0.012612800300121307
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,24,1,128,1,float16,float16,0,0.08021280169487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,24,1,128,1,float16,fp8,0,0.08356639742851257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,24,4,128,1,float16,fp8,0,0.013761599361896516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,24,1,128,1,fp8,fp8,0,0.08350560069084167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,24,2,128,1,float16,float16,0,0.08133440017700196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,24,2,128,1,float16,fp8,0,0.08423359990119934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,24,2,128,1,fp8,fp8,0,0.0847823977470398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,24,128,1,float16,fp8,0,0.014182400703430176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,24,4,128,1,float16,float16,0,0.10051840543746948
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,24,4,128,1,float16,fp8,0,0.10336159467697144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,24,1,128,1,float16,float16,0,0.017319999635219574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,24,8,128,1,float16,fp8,0,0.11032480001449585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,24,8,128,1,fp8,fp8,0,0.10927519798278809
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,24,128,1,float16,float16,0,0.13605120182037353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,24,128,1,float16,fp8,0,0.14283519983291626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,24,128,1,fp8,fp8,0,0.14345599412918092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,1,128,1,float16,float16,0,0.045582398772239685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,1,128,1,float16,fp8,0,0.04323999881744385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,1,128,1,fp8,fp8,0,0.04272800087928772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,2,128,1,float16,float16,0,0.04645279943943024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,2,128,1,float16,fp8,0,0.04431520104408264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,2,128,1,fp8,fp8,0,0.043540799617767335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,4,128,1,float16,float16,0,0.055936002731323244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,4,128,1,float16,fp8,0,0.055137598514556886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,4,128,1,fp8,fp8,0,0.0550383985042572
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,8,128,1,float16,float16,0,0.060303997993469236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,8,128,1,float16,fp8,0,0.058659201860427855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,24,8,128,1,fp8,fp8,0,0.059278398752212524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,24,128,1,float16,float16,0,0.07421280145645141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,24,128,1,float16,fp8,0,0.07421280145645141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,24,128,1,fp8,fp8,0,0.07356160283088684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,1,128,1,float16,float16,0,0.023478400707244874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,1,128,1,float16,fp8,0,0.024934400618076325
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,1,128,1,fp8,fp8,0,0.025166401267051698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,2,128,1,float16,float16,0,0.023391999304294586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,2,128,1,float16,fp8,0,0.025119999051094057
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,2,128,1,fp8,fp8,0,0.024940800666809083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,4,128,1,float16,float16,0,0.027694401144981385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,4,128,1,float16,fp8,0,0.029505598545074462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,4,128,1,fp8,fp8,0,0.02991040050983429
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,8,128,1,float16,float16,0,0.03047359883785248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,8,128,1,fp8,fp8,0,0.029814401268959047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,24,128,1,float16,float16,0,0.03734720051288605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,24,128,1,float16,fp8,0,0.03829280138015747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,24,128,1,fp8,fp8,0,0.038257598876953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,1,128,1,float16,float16,0,0.017667199671268462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,1,128,1,float16,fp8,0,0.01896799951791763
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,24,8,128,1,float16,float16,0,0.10896159410476684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,1,128,1,fp8,fp8,0,0.018943999707698823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,2,128,1,float16,float16,0,0.017841599881649017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,2,128,1,float16,fp8,0,0.01894560009241104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,24,24,128,1,fp8,fp8,0,0.0871616005897522
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,2,128,1,fp8,fp8,0,0.018908800184726716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,4,128,1,float16,float16,0,0.017772799730300902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,4,128,1,float16,fp8,0,0.018859200179576874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,8,128,1,float16,float16,0,0.018119999766349794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,8,128,1,float16,fp8,0,0.01891999989748001
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,8,128,1,fp8,fp8,0,0.019105599820613862
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,24,128,1,float16,float16,0,0.021716800332069398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,24,128,1,float16,fp8,0,0.0231904000043869
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,24,128,1,fp8,fp8,0,0.02311840057373047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,1,128,1,float16,float16,0,0.012939199805259705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,24,2,128,1,fp8,fp8,0,0.013036799430847169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,1,128,1,fp8,fp8,0,0.013635200262069703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,2,128,1,float16,float16,0,0.013225600123405457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,2,128,1,fp8,fp8,0,0.013604800403118133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,4,128,1,float16,float16,0,0.013044799864292144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,4,128,1,fp8,fp8,0,0.013630400598049163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,8,128,1,float16,float16,0,0.012996800243854523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,8,128,1,float16,fp8,0,0.013532799482345582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,8,128,1,fp8,fp8,0,0.013470399379730224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,24,128,1,float16,float16,0,0.017081600427627564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,24,128,1,float16,fp8,0,0.018185600638389587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,24,128,1,fp8,fp8,0,0.018171200156211854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,1,128,1,float16,float16,0,0.012942400574684144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,1,128,1,float16,fp8,0,0.013446399569511413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,24,8,128,1,float16,fp8,0,0.030419200658798218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,2,128,1,float16,float16,0,0.012627199292182922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,2,128,1,float16,fp8,0,0.013233600556850434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,2,128,1,fp8,fp8,0,0.013439999520778656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,4,128,1,float16,float16,0,0.012643200159072877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,24,24,128,1,fp8,fp8,0,0.013439999520778656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,4,128,1,float16,fp8,0,0.01321759968996048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,4,128,1,fp8,fp8,0,0.013123199343681335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,8,128,1,float16,float16,0,0.012814399600028992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,24,4,128,1,fp8,fp8,0,0.10337280035018921
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,8,128,1,fp8,fp8,0,0.013424000144004822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,24,128,1,float16,float16,0,0.01281599998474121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,24,128,1,float16,fp8,0,0.013648000359535218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,24,128,1,fp8,fp8,0,0.013329599797725678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,1,128,1,float16,float16,0,0.012375999987125397
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,1,128,1,float16,fp8,0,0.012998400628566742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,1,128,1,fp8,fp8,0,0.013107199966907502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,24,4,128,1,fp8,fp8,0,0.019073599576950075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,2,128,1,float16,fp8,0,0.01311040073633194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,2,128,1,fp8,fp8,0,0.012811200320720672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,4,128,1,float16,fp8,0,0.012851199507713318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,4,128,1,fp8,fp8,0,0.012838399410247803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,1,128,1,float16,fp8,0,0.01348000019788742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,8,128,1,float16,float16,0,0.012185599654912949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,8,128,1,float16,fp8,0,0.012972800433635712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,4,128,1,float16,fp8,0,0.013579200208187103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,24,128,1,float16,float16,0,0.01228640004992485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,24,128,1,float16,fp8,0,0.013043199479579926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,24,128,1,fp8,fp8,0,0.013198399543762207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,1,128,1,float16,float16,0,0.012115199863910676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,1,128,1,float16,fp8,0,0.012758399546146392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,1,128,1,fp8,fp8,0,0.01276959925889969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,2,128,1,float16,float16,0,0.012111999839544297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,2,128,1,float16,fp8,0,0.012745599448680877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,2,128,1,fp8,fp8,0,0.012638400495052337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,4,128,1,float16,float16,0,0.011955200135707856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,1,128,1,fp8,fp8,0,0.013339200615882873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,4,128,1,fp8,fp8,0,0.01271360069513321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,8,128,1,float16,float16,0,0.01180960014462471
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,8,128,1,float16,fp8,0,0.012758399546146392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,8,128,1,fp8,fp8,0,0.01255040019750595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,24,128,1,float16,float16,0,0.012382400035858155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,24,128,1,float16,fp8,0,0.013332800567150116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,24,128,1,fp8,fp8,0,0.0133200004696846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,24,8,128,1,float16,fp8,0,0.013094399869441987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,1,128,1,float16,float16,0,0.01199359968304634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,1,128,1,float16,fp8,0,0.012721599638462066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,1,128,1,fp8,fp8,0,0.012758399546146392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,2,128,1,float16,float16,0,0.01202400028705597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,2,128,1,fp8,fp8,0,0.012726399302482604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,4,128,1,float16,float16,0,0.012027200311422348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,4,128,1,float16,fp8,0,0.012694400548934937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,4,128,1,fp8,fp8,0,0.012611199915409089
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,8,128,1,float16,float16,0,0.01175839975476265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,24,2,128,1,float16,fp8,0,0.013660800457000733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,8,128,1,fp8,fp8,0,0.012542399764060973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,8,128,1,fp8,fp8,0,0.012943999469280243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,24,1,128,1,float16,float16,0,0.047092801332473753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,24,1,128,1,float16,fp8,0,0.04480159878730774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,24,1,128,1,fp8,fp8,0,0.0445360004901886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,24,2,128,1,float16,float16,0,0.0546671986579895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,24,2,128,1,float16,fp8,0,0.054369598627090454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,24,2,128,1,fp8,fp8,0,0.05451520085334778
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,24,4,128,1,float16,float16,0,0.05787519812583923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,24,4,128,1,float16,fp8,0,0.0566976010799408
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,24,4,128,1,fp8,fp8,0,0.056878399848937986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,24,8,128,1,float16,float16,0,0.09155359864234924
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,24,8,128,1,float16,fp8,0,0.09570720195770263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,24,8,128,1,fp8,fp8,0,0.09723680019378662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,24,4,128,1,float16,fp8,0,0.012627199292182922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,24,128,1,float16,fp8,0,0.12946560382843017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,24,128,1,fp8,fp8,0,0.13036799430847168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,1,128,1,float16,float16,0,0.023656000196933747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,1,128,1,float16,fp8,0,0.02640799880027771
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,1,128,1,fp8,fp8,0,0.02600800096988678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,2,128,1,float16,float16,0,0.027819201350212097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,2,128,1,fp8,fp8,0,0.03084160089492798
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,4,128,1,float16,float16,0,0.028548800945281984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,4,128,1,float16,fp8,0,0.030670401453971863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,4,128,1,fp8,fp8,0,0.03078559935092926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,2,128,1,float16,float16,0,0.01223199963569641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,24,4,128,1,float16,float16,0,0.012185599654912949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,8,128,1,float16,fp8,0,0.048758399486541745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,8,128,1,fp8,fp8,0,0.04895519912242889
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,24,128,1,float16,float16,0,0.059297597408294676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,24,128,1,float16,fp8,0,0.0663375973701477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,24,128,1,fp8,fp8,0,0.06629279851913453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,8,128,1,float16,fp8,0,0.012427199631929398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,1,128,1,float16,float16,0,0.018705600500106813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,1,128,1,float16,fp8,0,0.019787199795246124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,1,128,1,fp8,fp8,0,0.019494399428367615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,2,128,1,float16,float16,0,0.018220800161361694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,2,128,1,float16,fp8,0,0.019787199795246124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,2,128,1,fp8,fp8,0,0.01932159960269928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,4,128,1,float16,float16,0,0.018680000305175783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,4,128,1,fp8,fp8,0,0.019457599520683287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,8,128,1,float16,float16,0,0.026289600133895873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,8,128,1,float16,fp8,0,0.028553599119186403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,8,128,1,fp8,fp8,0,0.028366398811340333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,24,128,1,float16,fp8,0,0.03737919926643372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,24,128,1,fp8,fp8,0,0.037257599830627444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,1,128,1,float16,float16,0,0.013283200562000275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,1,128,1,float16,fp8,0,0.014004799723625182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,1,128,1,fp8,fp8,0,0.014236800372600555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,2,128,1,float16,float16,0,0.013233600556850434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,2,128,1,float16,fp8,0,0.014126400649547576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,2,128,1,fp8,fp8,0,0.01404159963130951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,4,128,1,float16,float16,0,0.013126400113105775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,4,128,1,float16,fp8,0,0.013739199936389923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,4,128,1,fp8,fp8,0,0.013849599659442902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,8,128,1,float16,float16,0,0.017057600617408752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,8,128,1,float16,fp8,0,0.01845120042562485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,8,128,1,fp8,fp8,0,0.01833280026912689
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,24,128,1,float16,float16,0,0.12147200107574463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,24,128,1,float16,fp8,0,0.02285439968109131
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,24,128,1,fp8,fp8,0,0.022702400386333466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,1,128,1,float16,fp8,0,0.013398399949073792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,1,128,1,fp8,fp8,0,0.013359999656677246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,2,128,1,float16,fp8,0,0.030432000756263733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,2,128,1,float16,fp8,0,0.013118399679660797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,2,128,1,fp8,fp8,0,0.013153600692749023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,4,128,1,float16,float16,0,0.012476799637079239
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,24,2,128,1,float16,fp8,0,0.012724800407886505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,24,8,128,1,float16,float16,0,0.046177598834037784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,4,128,1,fp8,fp8,0,0.013382400572299957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,8,128,1,float16,float16,0,0.012772800028324127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,8,128,1,float16,fp8,0,0.013443200290203095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,8,128,1,fp8,fp8,0,0.013358399271965027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,24,128,1,float16,float16,0,0.016574400663375854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,24,128,1,float16,fp8,0,0.017671999335289002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,24,4,128,1,float16,fp8,0,0.019859200716018675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,24,128,1,fp8,fp8,0,0.017668800055980684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,1,128,1,float16,float16,0,0.012537600100040435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,1,128,1,fp8,fp8,0,0.013123199343681335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,2,128,1,float16,float16,0,0.01233920007944107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,2,128,1,float16,fp8,0,0.01308799982070923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,2,128,1,fp8,fp8,0,0.013214400410652161
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,4,128,1,float16,float16,0,0.012444800138473511
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,4,128,1,float16,fp8,0,0.013204799592494964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,4,128,1,fp8,fp8,0,0.012988799810409546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,8,128,1,float16,float16,0,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,8,128,1,float16,fp8,0,0.012985600531101227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,1,128,1,float16,float16,0,0.012615999579429627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,24,128,1,float16,float16,0,0.012591999769210816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,24,128,1,float16,fp8,0,0.01345600038766861
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,24,128,1,fp8,fp8,0,0.013385599851608277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,1,128,1,float16,float16,0,0.011926399916410447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,1,128,1,float16,fp8,0,0.0126351997256279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,1,128,1,fp8,fp8,0,0.012643200159072877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,2,128,1,float16,float16,0,0.011894399672746659
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,2,128,1,float16,fp8,0,0.012548799812793731
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,2,128,1,fp8,fp8,0,0.01271360069513321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,4,128,1,float16,fp8,0,0.013278399407863618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,4,128,1,float16,float16,0,0.01196800023317337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,4,128,1,float16,fp8,0,0.012454400211572647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,8,128,1,float16,float16,0,0.011964800208806992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,8,128,1,float16,fp8,0,0.012718400359153748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,8,128,1,fp8,fp8,0,0.012814399600028992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,24,128,1,float16,float16,0,0.012478400021791458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,24,128,1,float16,fp8,0,0.013155199587345123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,24,128,1,fp8,fp8,0,0.013172799348831176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,1,128,1,float16,float16,0,0.011870399862527848
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,1,128,1,float16,fp8,0,0.012591999769210816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,1,128,1,fp8,fp8,0,0.01257600039243698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,2,128,1,float16,float16,0,0.012095999717712403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,2,128,1,float16,fp8,0,0.012783999741077422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,2,128,1,fp8,fp8,0,0.01273919939994812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,4,128,1,float16,float16,0,0.011975999921560287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,4,128,1,float16,fp8,0,0.012641599774360657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,4,128,1,fp8,fp8,0,0.01249919980764389
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,8,128,1,float16,float16,0,0.011876799911260606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,8,128,1,float16,fp8,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,24,8,128,1,fp8,fp8,0,0.012595200538635254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,24,128,1,float16,float16,0,0.012177599966526032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,24,128,1,float16,fp8,0,0.013076800107955932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,24,128,1,fp8,fp8,0,0.0131632000207901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,1,128,1,float16,float16,0,0.011919999867677689
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,1,128,1,float16,fp8,0,0.013129599392414093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,1,128,1,fp8,fp8,0,0.012596799433231354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,2,128,1,float16,float16,0,0.01180799975991249
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,2,128,1,float16,fp8,0,0.012521600723266602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,2,128,1,fp8,fp8,0,0.012491200119256973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,4,128,1,float16,float16,0,0.011590400338172912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,4,128,1,float16,fp8,0,0.012328000366687774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,24,4,128,1,fp8,fp8,0,0.012753599882125854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,8,128,1,float16,float16,0,0.011748799681663513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,8,128,1,float16,fp8,0,0.012462399899959564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,8,128,1,fp8,fp8,0,0.012483199685811996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,24,1,128,1,float16,float16,0,0.020790399610996248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,24,1,128,1,float16,fp8,0,0.02292799949645996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,24,1,128,1,fp8,fp8,0,0.022300800681114195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,24,2,128,1,float16,float16,0,0.028376001119613647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,24,2,128,1,float16,fp8,0,0.031004801392555237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,24,2,128,1,fp8,fp8,0,0.03142080008983612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,24,4,128,1,float16,float16,0,0.04307680130004883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,24,4,128,1,float16,fp8,0,0.049089598655700686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,24,4,128,1,fp8,fp8,0,0.04892959892749786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,24,24,128,1,float16,float16,0,0.03395360112190247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,24,8,128,1,float16,float16,0,0.07133280038833618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,24,8,128,1,float16,fp8,0,0.0842415988445282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,24,8,128,1,fp8,fp8,0,0.08464000225067139
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,24,128,1,float16,fp8,0,0.11941280364990234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,24,128,1,fp8,fp8,0,0.12004319429397584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,1,128,1,float16,fp8,0,0.015835200250148774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,1,128,1,fp8,fp8,0,0.015884800255298613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,2,128,1,float16,float16,0,0.018745599687099455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,24,128,1,float16,float16,0,0.020923200249671935
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,2,128,1,fp8,fp8,0,0.02032800018787384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,8,128,1,fp8,fp8,0,0.013030399382114411
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,4,128,1,float16,float16,0,0.026067200303077697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,4,128,1,float16,fp8,0,0.02902719974517822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,4,128,1,fp8,fp8,0,0.02911840081214905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,24,2,128,1,float16,float16,0,0.012374400347471236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,8,128,1,fp8,fp8,0,0.046433600783348086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,24,128,1,float16,float16,0,0.05469120144844055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,24,128,1,float16,fp8,0,0.06428160071372986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,24,128,1,fp8,fp8,0,0.06432960033416749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,1,128,1,float16,float16,0,0.013225600123405457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,24,4,128,1,fp8,fp8,0,0.012300799787044524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,1,128,1,fp8,fp8,0,0.014137600362300873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,2,128,1,float16,float16,0,0.013407999277114868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,2,128,1,float16,fp8,0,0.014336000382900237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,24,1,128,1,float16,fp8,0,0.01311199963092804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,2,128,1,fp8,fp8,0,0.014030399918556213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,4,128,1,float16,float16,0,0.016991999745368958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,24,128,1,float16,float16,0,0.09860960245132447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,4,128,1,float16,fp8,0,0.01855680048465729
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,4,128,1,fp8,fp8,0,0.018478399515151976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,1,128,1,float16,float16,0,0.014507199823856353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,8,128,1,float16,fp8,0,0.02731199860572815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,8,128,1,fp8,fp8,0,0.027486398816108704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,24,128,1,float16,float16,0,0.03242720067501068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,24,128,1,float16,fp8,0,0.03628320097923279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,24,128,1,fp8,fp8,0,0.036652800440788266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,1,128,1,float16,float16,0,0.012350399792194367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,1,128,1,float16,fp8,0,0.013555200397968292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,1,128,1,fp8,fp8,0,0.013348799943923951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,2,128,1,float16,float16,0,0.012798400223255157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,2,128,1,float16,fp8,0,0.013390399515628815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,2,128,1,fp8,fp8,0,0.01356479972600937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,4,128,1,float16,fp8,0,0.013540799915790557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,4,128,1,fp8,fp8,0,0.013486400246620178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,8,128,1,float16,float16,0,0.016564799845218657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,8,128,1,float16,fp8,0,0.04659520089626312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,8,128,1,float16,fp8,0,0.01785759925842285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,8,128,1,fp8,fp8,0,0.017907199263572694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,24,128,1,float16,fp8,0,0.02234079986810684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,24,128,1,fp8,fp8,0,0.022438399493694305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,1,128,1,float16,float16,0,0.01204800009727478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,1,128,1,float16,fp8,0,0.012851199507713318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,1,128,1,fp8,fp8,0,0.012825599312782288
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,1,128,1,float16,fp8,0,0.014060799777507783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,2,128,1,float16,float16,0,0.012254399806261062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,2,128,1,float16,fp8,0,0.012811200320720672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,4,128,1,float16,float16,0,0.01194079965353012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,4,128,1,float16,fp8,0,0.012796799838542938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,4,128,1,fp8,fp8,0,0.012774400413036346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,8,128,1,float16,float16,0,0.012164799869060517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,8,128,1,float16,fp8,0,0.013017599284648896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,8,128,1,fp8,fp8,0,0.013043199479579926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,24,128,1,float16,float16,0,0.016313600540161132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,24,128,1,float16,fp8,0,0.017601600289344786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,24,128,1,fp8,fp8,0,0.01757120043039322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,1,128,1,float16,float16,0,0.011539199948310852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,1,128,1,float16,fp8,0,0.012355200201272964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,24,8,128,1,float16,float16,0,0.024460799992084503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,1,128,1,fp8,fp8,0,0.01257600039243698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,2,128,1,float16,fp8,0,0.012745599448680877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,2,128,1,fp8,fp8,0,0.012721599638462066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,4,128,1,float16,float16,0,0.012219200283288956
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,4,128,1,float16,fp8,0,0.012774400413036346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,4,128,1,fp8,fp8,0,0.012972800433635712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,8,128,1,float16,float16,0,0.012094400078058242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,8,128,1,float16,fp8,0,0.012886400520801543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,2,128,1,float16,fp8,0,0.020295999944210052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,8,128,1,fp8,fp8,0,0.012833599746227265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,24,128,1,float16,float16,0,0.012227199971675873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,24,128,1,float16,fp8,0,0.013380800187587739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,1,128,1,float16,float16,0,0.011388800293207168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,24,4,128,1,float16,float16,0,0.012675200402736665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,1,128,1,fp8,fp8,0,0.012272000312805176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,2,128,1,float16,float16,0,0.011512000113725662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,2,128,1,float16,fp8,0,0.012305600196123123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,2,128,1,fp8,fp8,0,0.01223360002040863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,4,128,1,float16,float16,0,0.011788800358772278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,24,8,128,1,float16,float16,0,0.04073759913444519
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,4,128,1,fp8,fp8,0,0.012377600371837615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,8,128,1,float16,float16,0,0.0119439996778965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,8,128,1,float16,fp8,0,0.012548799812793731
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,24,128,1,float16,float16,0,0.020326399803161622
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,24,128,1,float16,float16,0,0.012076800316572189
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,24,128,1,float16,fp8,0,0.01300320029258728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,24,128,1,fp8,fp8,0,0.012889599800109864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,1,128,1,float16,float16,0,0.011523199826478958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,1,128,1,fp8,fp8,0,0.012319999933242797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,2,128,1,float16,float16,0,0.011521600186824799
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,2,128,1,float16,fp8,0,0.012281599640846252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,2,128,1,fp8,fp8,0,0.012280000001192093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,24,2,128,1,fp8,fp8,0,0.013036799430847169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,4,128,1,float16,fp8,0,0.0121568001806736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,4,128,1,fp8,fp8,0,0.012211199849843979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,8,128,1,float16,float16,0,0.011473599821329117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,8,128,1,float16,fp8,0,0.012692800164222718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,8,128,1,fp8,fp8,0,0.01223199963569641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,24,128,1,float16,float16,0,0.012359999865293504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,24,128,1,float16,fp8,0,0.013115200400352477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,24,128,1,fp8,fp8,0,0.01313440054655075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,1,128,1,float16,float16,0,0.01138240024447441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,1,128,1,float16,fp8,0,0.012151999771595002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,1,128,1,fp8,fp8,0,0.012319999933242797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,24,2,128,1,float16,float16,0,0.011751999706029892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,2,128,1,float16,fp8,0,0.011977600306272507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,2,128,1,fp8,fp8,0,0.012078399956226348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,4,128,1,float16,float16,0,0.011497599631547928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,4,128,1,float16,fp8,0,0.012249600142240524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,24,128,1,fp8,fp8,0,0.013123199343681335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,1,128,1,float16,fp8,0,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,8,128,1,float16,float16,0,0.01148959994316101
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,8,128,1,float16,fp8,0,0.012319999933242797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,8,128,1,fp8,fp8,0,0.012361600250005721
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,4,128,1,float16,fp8,0,0.012308800220489502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,24,8,128,1,fp8,fp8,0,0.0127920001745224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,1,128,1,float16,fp8,0,0.012076800316572189
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,24,4,128,1,float16,float16,0,0.011404799669981003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,2,128,1,float16,float16,0,0.011510399729013443
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,24,4,128,1,fp8,fp8,0,0.012374400347471236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,16,1,128,1,float16,fp8,0,12.764806365966797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,16,1,128,1,fp8,fp8,0,12.94439239501953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,16,2,128,1,float16,fp8,0,12.670919799804688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,16,1,128,1,float16,float16,0,15.124685668945313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,16,2,128,1,fp8,fp8,0,12.776211547851563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,16,2,128,1,float16,float16,0,15.266154479980468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,16,4,128,1,float16,float16,0,15.667620849609374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,16,4,128,1,float16,fp8,0,12.892556762695312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,16,128,1,float16,fp8,0,6.56590576171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,16,128,1,float16,float16,0,7.877188873291016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,16,4,128,1,fp8,fp8,0,12.797617340087891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,16,128,1,fp8,fp8,0,6.664889526367188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,16,8,128,1,float16,fp8,0,12.815910339355469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,1,128,1,float16,float16,0,7.709069061279297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,16,8,128,1,fp8,fp8,0,12.923918151855469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,16,8,128,1,float16,float16,0,15.76593475341797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,1,128,1,float16,fp8,0,6.403826904296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,1,128,1,fp8,fp8,0,6.423766326904297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,2,128,1,float16,fp8,0,6.464892578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,2,128,1,float16,float16,0,7.688382720947265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,2,128,1,fp8,fp8,0,6.361982345581055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,4,128,1,float16,fp8,0,6.585775756835938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,4,128,1,float16,float16,0,7.786373138427734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,4,128,1,fp8,fp8,0,7.062248229980469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,8,128,1,float16,fp8,0,6.522051239013672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,16,128,1,float16,fp8,0,3.6246097564697264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,8,128,1,float16,float16,0,8.546943664550781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,16,128,1,float16,float16,0,4.407115173339844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,16,128,1,fp8,fp8,0,3.3647201538085936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,1,128,1,float16,fp8,0,3.2872974395751955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,16,8,128,1,fp8,fp8,0,7.09441909790039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,1,128,1,float16,float16,0,3.919148635864258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,1,128,1,fp8,fp8,0,3.382097625732422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,2,128,1,float16,fp8,0,3.310555267333984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,2,128,1,float16,float16,0,3.9645137786865234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,2,128,1,fp8,fp8,0,3.368657684326172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,4,128,1,float16,float16,0,3.6610111236572265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,4,128,1,float16,fp8,0,3.2456302642822266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,4,128,1,fp8,fp8,0,3.282099151611328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,16,128,1,float16,fp8,0,1.629145622253418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,16,128,1,float16,float16,0,1.9763055801391602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,8,128,1,float16,fp8,0,3.250320053100586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,8,128,1,float16,float16,0,4.443033599853516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,16,128,1,fp8,fp8,0,2.1101247787475588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,16,8,128,1,fp8,fp8,0,3.232516860961914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,1,128,1,float16,float16,0,1.7342191696166993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,1,128,1,float16,fp8,0,1.6000127792358398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,1,128,1,fp8,fp8,0,1.5800432205200194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,2,128,1,float16,float16,0,1.877694320678711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,2,128,1,float16,fp8,0,1.5926655769348144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,4,128,1,float16,fp8,0,1.6080480575561524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,4,128,1,float16,float16,0,1.7740608215332032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,2,128,1,fp8,fp8,0,2.041579246520996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,4,128,1,fp8,fp8,0,1.8135856628417968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,8,128,1,float16,fp8,0,1.5921728134155273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,8,128,1,fp8,fp8,0,1.7299360275268554
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,16,8,128,1,float16,float16,0,1.8432912826538086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,16,1,128,1,float16,fp8,0,7.307895660400391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,16,1,128,1,fp8,fp8,0,7.347281646728516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,16,2,128,1,float16,fp8,0,7.3743408203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,16,2,128,1,fp8,fp8,0,7.267660522460938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,16,1,128,1,float16,float16,0,8.756969451904297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,16,2,128,1,float16,float16,0,8.865734100341797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,16,4,128,1,float16,float16,0,8.66052017211914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,16,4,128,1,float16,fp8,0,7.389627075195312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,16,128,1,float16,float16,0,4.5724224090576175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,16,128,1,float16,fp8,0,3.875377655029297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,16,128,1,fp8,fp8,0,3.8322063446044923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,16,4,128,1,fp8,fp8,0,7.406043243408203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,16,8,128,1,fp8,fp8,0,7.357389068603515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,16,8,128,1,float16,fp8,0,7.487254333496094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,1,128,1,float16,float16,0,4.323076629638672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,16,8,128,1,float16,float16,0,9.046417236328125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,1,128,1,float16,fp8,0,3.7333518981933596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,2,128,1,float16,float16,0,4.271420669555664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,2,128,1,float16,fp8,0,3.6348400115966797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,2,128,1,fp8,fp8,0,3.611920166015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,4,128,1,float16,float16,0,4.61059341430664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,4,128,1,float16,fp8,0,3.8562015533447265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,4,128,1,fp8,fp8,0,3.822003173828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,8,128,1,float16,float16,0,4.442827224731445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,16,128,1,float16,float16,0,2.1687231063842773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,8,128,1,float16,fp8,0,3.7844318389892577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,16,128,1,fp8,fp8,0,1.8732303619384765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,16,128,1,float16,fp8,0,2.0618480682373046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,8,128,1,fp8,fp8,0,3.696625518798828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,1,128,1,float16,float16,0,2.036403274536133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,1,128,1,float16,fp8,0,1.9385120391845703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,1,128,1,fp8,fp8,0,1.9636463165283202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,2,128,1,float16,fp8,0,1.7967199325561523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,2,128,1,fp8,fp8,0,1.8933328628540038
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,4,128,1,float16,float16,0,2.1480464935302734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,4,128,1,float16,fp8,0,2.099492835998535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,4,128,1,fp8,fp8,0,1.8281536102294922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,8,128,1,float16,float16,0,2.1022863388061523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,16,128,1,float16,float16,0,1.2662912368774415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,16,128,1,float16,fp8,0,0.9769488334655761
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,8,128,1,float16,fp8,0,1.850726318359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,8,128,1,fp8,fp8,0,1.9040191650390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,16,128,1,fp8,fp8,0,0.9641983985900879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,1,128,1,float16,fp8,0,0.9209712028503418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,1,128,1,float16,float16,0,1.2672752380371093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,2,128,1,float16,fp8,0,0.9007792472839355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,2,128,1,float16,float16,0,0.9771280288696289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,16,2,128,1,float16,float16,0,2.021447944641113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,2,128,1,fp8,fp8,0,0.9035231590270996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,4,128,1,float16,fp8,0,0.9058496475219726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,4,128,1,fp8,fp8,0,0.9446335792541504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,8,128,1,float16,float16,0,1.0004256248474122
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,8,128,1,float16,fp8,0,0.9123600006103516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,8,128,1,fp8,fp8,0,0.9237872123718261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,1,128,1,fp8,fp8,0,0.9006352424621582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,16,1,128,1,fp8,fp8,0,3.646865463256836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,16,4,128,1,float16,float16,0,1.2199664115905762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,16,1,128,1,float16,fp8,0,5.067270278930664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,16,1,128,1,fp8,fp8,0,5.085696029663086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,16,1,128,1,float16,float16,0,6.44292984008789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,16,2,128,1,float16,float16,0,6.243212890625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,16,2,128,1,float16,fp8,0,5.133430480957031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,16,2,128,1,fp8,fp8,0,5.0818431854248045
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,16,4,128,1,float16,fp8,0,5.128580856323242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,16,4,128,1,float16,float16,0,6.004304122924805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,16,128,1,float16,float16,0,2.9746896743774416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,16,4,128,1,fp8,fp8,0,5.088264083862304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,16,128,1,float16,fp8,0,2.656635284423828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,16,8,128,1,float16,fp8,0,5.327582550048828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,16,128,1,fp8,fp8,0,2.6566768646240235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,16,8,128,1,float16,float16,0,6.57079849243164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,16,8,128,1,fp8,fp8,0,5.2395263671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,1,128,1,float16,float16,0,2.957846450805664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,1,128,1,fp8,fp8,0,2.5273920059204102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,1,128,1,float16,fp8,0,2.6752639770507813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,2,128,1,float16,float16,0,2.948294448852539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,2,128,1,float16,fp8,0,2.5683231353759766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,2,128,1,fp8,fp8,0,2.563369560241699
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,4,128,1,float16,float16,0,3.014462471008301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,4,128,1,float16,fp8,0,2.542296028137207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,4,128,1,fp8,fp8,0,2.653816032409668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,16,128,1,float16,fp8,0,1.339961624145508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,16,128,1,float16,float16,0,1.5443120002746582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,8,128,1,float16,fp8,0,2.6080703735351562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,8,128,1,float16,float16,0,3.0054576873779295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,1,128,1,float16,float16,0,1.3748463630676269
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,16,128,1,fp8,fp8,0,1.4768511772155761
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,16,8,128,1,fp8,fp8,0,2.754055976867676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,1,128,1,fp8,fp8,0,1.2560272216796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,2,128,1,float16,fp8,0,1.2763680458068847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,2,128,1,float16,float16,0,1.5157072067260742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,2,128,1,fp8,fp8,0,1.4053104400634766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,4,128,1,float16,fp8,0,1.3367631912231446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,4,128,1,float16,float16,0,1.4944031715393067
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,4,128,1,fp8,fp8,0,1.283796787261963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,16,128,1,float16,float16,0,0.7777679920196533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,8,128,1,float16,fp8,0,1.2934720039367675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,16,128,1,float16,fp8,0,0.7170239925384522
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,16,128,1,fp8,fp8,0,0.8251008033752442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,1,128,1,float16,float16,0,0.6887919902801514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,1,128,1,float16,fp8,0,0.650932788848877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,1,128,1,fp8,fp8,0,0.6378111839294434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,2,128,1,float16,float16,0,0.6803823947906494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,2,128,1,fp8,fp8,0,0.635745620727539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,4,128,1,float16,float16,0,0.6915631771087647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,1,128,1,float16,fp8,0,1.2723376274108886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,4,128,1,fp8,fp8,0,0.6540063858032227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,8,128,1,float16,float16,0,0.7301599979400635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,8,128,1,float16,fp8,0,0.6639823913574219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,8,128,1,fp8,fp8,0,0.6536784172058105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,8,128,1,float16,float16,0,1.4425472259521483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,16,8,128,1,fp8,fp8,0,1.403825569152832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,2,128,1,float16,fp8,0,0.7135600090026856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,16,4,128,1,float16,fp8,0,0.6387743949890137
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,16,1,128,1,float16,fp8,0,6.718780517578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,16,1,128,1,fp8,fp8,0,6.619561767578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,16,1,128,1,float16,float16,0,7.9481651306152346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,16,2,128,1,float16,float16,0,7.8543647766113285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,16,2,128,1,float16,fp8,0,6.66723861694336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,16,2,128,1,fp8,fp8,0,6.697652435302734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,16,4,128,1,float16,fp8,0,6.729891204833985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,16,4,128,1,float16,float16,0,7.87974853515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,16,128,1,float16,fp8,0,3.5122608184814452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,16,128,1,float16,float16,0,4.127652740478515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,16,4,128,1,fp8,fp8,0,6.753610992431641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,16,128,1,fp8,fp8,0,3.488391876220703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,1,128,1,float16,float16,0,3.8983551025390626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,16,8,128,1,fp8,fp8,0,6.805699157714844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,16,8,128,1,float16,float16,0,8.718911743164062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,1,128,1,float16,fp8,0,3.3578624725341797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,1,128,1,fp8,fp8,0,3.4190288543701173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,2,128,1,float16,fp8,0,3.3444080352783203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,2,128,1,float16,float16,0,4.544478225708008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,2,128,1,fp8,fp8,0,3.4352127075195313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,16,8,128,1,float16,fp8,0,6.9076080322265625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,4,128,1,float16,float16,0,3.8823089599609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,4,128,1,fp8,fp8,0,3.3606990814208983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,8,128,1,float16,float16,0,4.011590576171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,16,128,1,float16,fp8,0,1.7126335144042968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,16,128,1,float16,float16,0,1.9630256652832032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,8,128,1,float16,fp8,0,3.404657745361328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,16,128,1,fp8,fp8,0,1.8667760848999024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,1,128,1,float16,float16,0,1.8505504608154297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,8,128,1,fp8,fp8,0,3.5942977905273437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,1,128,1,float16,fp8,0,1.8307792663574218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,1,128,1,fp8,fp8,0,1.7728544235229493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,2,128,1,float16,float16,0,1.854360008239746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,2,128,1,float16,fp8,0,1.6711584091186524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,16,4,128,1,float16,fp8,0,3.553731155395508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,2,128,1,fp8,fp8,0,1.778152084350586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,4,128,1,float16,float16,0,1.8729999542236329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,4,128,1,float16,fp8,0,1.6784608840942383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,4,128,1,fp8,fp8,0,1.6781871795654297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,16,128,1,float16,float16,0,0.9696304321289062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,8,128,1,float16,fp8,0,1.6727392196655273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,8,128,1,float16,float16,0,2.0372447967529297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,16,8,128,1,fp8,fp8,0,1.674603271484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,16,128,1,fp8,fp8,0,0.8810031890869141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,1,128,1,float16,float16,0,0.8816240310668946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,1,128,1,float16,fp8,0,0.9135536193847656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,1,128,1,fp8,fp8,0,0.8383760452270508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,2,128,1,float16,fp8,0,0.8267616271972656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,2,128,1,float16,float16,0,0.9817855834960938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,4,128,1,float16,float16,0,0.9694479942321778
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,4,128,1,float16,fp8,0,0.8620335578918457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,4,128,1,fp8,fp8,0,0.8357135772705078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,16,128,1,float16,float16,0,0.48024959564208985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,8,128,1,float16,fp8,0,0.9419504165649414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,8,128,1,fp8,fp8,0,0.9311039924621582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,16,128,1,float16,fp8,0,0.4425312042236328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,16,128,1,fp8,fp8,0,0.4712831974029541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,1,128,1,float16,fp8,0,0.4418447971343994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,1,128,1,fp8,fp8,0,0.4347504138946533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,2,128,1,float16,float16,0,0.44878559112548827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,2,128,1,float16,fp8,0,0.435316801071167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,4,128,1,float16,float16,0,0.4561583995819092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,16,128,1,float16,fp8,0,0.9694383621215821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,8,128,1,float16,float16,0,0.4624368190765381
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,8,128,1,float16,fp8,0,0.43223037719726565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,8,128,1,fp8,fp8,0,0.4313551902770996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,2,128,1,fp8,fp8,0,0.8280415534973145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,16,8,128,1,float16,float16,0,0.9100655555725098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,1,128,1,float16,float16,0,0.4426527976989746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,2,128,1,fp8,fp8,0,0.433457612991333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,4,128,1,float16,fp8,0,0.4225791931152344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,16,4,128,1,fp8,fp8,0,0.43620800971984863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,16,1,128,1,float16,fp8,0,3.83702392578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,16,1,128,1,fp8,fp8,0,3.8042976379394533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,16,1,128,1,float16,float16,0,4.429971313476562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,16,2,128,1,float16,float16,0,4.3310302734375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,16,2,128,1,float16,fp8,0,3.7720256805419923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,16,2,128,1,fp8,fp8,0,3.8351776123046877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,16,4,128,1,float16,fp8,0,3.831982421875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,16,4,128,1,float16,float16,0,4.426601409912109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,16,4,128,1,fp8,fp8,0,4.024411010742187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,16,128,1,float16,float16,0,2.3415504455566407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,16,8,128,1,float16,fp8,0,4.066136169433594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,16,8,128,1,float16,float16,0,4.687247848510742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,16,128,1,fp8,fp8,0,2.029574394226074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,1,128,1,float16,float16,0,2.2316991806030275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,16,8,128,1,fp8,fp8,0,3.996094512939453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,1,128,1,float16,fp8,0,1.9326976776123046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,1,128,1,fp8,fp8,0,2.0219615936279296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,2,128,1,float16,float16,0,2.120604705810547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,2,128,1,float16,fp8,0,2.0410255432128905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,2,128,1,fp8,fp8,0,1.921748733520508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,4,128,1,float16,float16,0,2.249091148376465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,4,128,1,float16,fp8,0,1.9288415908813477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,4,128,1,fp8,fp8,0,1.9079456329345703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,16,128,1,float16,fp8,0,2.026815986633301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,16,128,1,float16,float16,0,1.1466879844665527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,8,128,1,float16,float16,0,2.1122655868530273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,16,128,1,fp8,fp8,0,1.014628791809082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,8,128,1,float16,fp8,0,2.1016799926757814
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,1,128,1,float16,float16,0,1.0158191680908204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,1,128,1,float16,fp8,0,0.9585871696472168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,1,128,1,fp8,fp8,0,1.1025072097778321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,2,128,1,float16,float16,0,1.0688688278198242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,2,128,1,float16,fp8,0,0.9570207595825195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,2,128,1,fp8,fp8,0,0.9803952217102051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,4,128,1,float16,fp8,0,0.9584112167358398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,4,128,1,float16,float16,0,1.112880039215088
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,4,128,1,fp8,fp8,0,1.0250911712646484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,8,128,1,float16,float16,0,1.1650464057922363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,8,128,1,fp8,fp8,0,0.9807791709899902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,16,128,1,fp8,fp8,0,0.524832010269165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,1,128,1,float16,float16,0,0.565388822555542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,1,128,1,fp8,fp8,0,0.4848288059234619
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,2,128,1,float16,float16,0,0.5476943969726562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,2,128,1,float16,fp8,0,0.4805568218231201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,16,128,1,float16,fp8,0,1.1246864318847656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,16,8,128,1,fp8,fp8,0,1.9673263549804687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,2,128,1,fp8,fp8,0,0.5174032211303711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,4,128,1,float16,float16,0,0.522486400604248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,4,128,1,float16,fp8,0,0.48825759887695314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,4,128,1,fp8,fp8,0,0.48409762382507326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,16,128,1,float16,float16,0,0.30813119411468504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,8,128,1,float16,float16,0,0.5475552082061768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,8,128,1,fp8,fp8,0,0.4967872142791748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,16,128,1,float16,fp8,0,0.26690239906311036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,16,128,1,fp8,fp8,0,0.27503840923309325
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,1,128,1,float16,float16,0,0.2595200061798096
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,1,128,1,float16,fp8,0,0.24736320972442627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,16,8,128,1,float16,fp8,0,0.9769935607910156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,1,128,1,fp8,fp8,0,0.24519040584564208
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,2,128,1,float16,float16,0,0.26357600688934324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,2,128,1,float16,fp8,0,0.2465967893600464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,2,128,1,fp8,fp8,0,0.24579839706420897
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,4,128,1,float16,float16,0,0.26672959327697754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,1,128,1,float16,fp8,0,0.4836895942687988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,4,128,1,float16,fp8,0,0.24835679531097413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,4,128,1,fp8,fp8,0,0.24804320335388183
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,8,128,1,float16,float16,0,0.27942719459533694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,8,128,1,float16,fp8,0,0.25217280387878416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,16,8,128,1,fp8,fp8,0,0.25486719608306885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,8,128,1,float16,fp8,0,0.4922319889068604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,16,128,1,float16,float16,0,0.5937664031982421
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,16,1,128,1,float16,fp8,0,3.568435287475586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,16,1,128,1,fp8,fp8,0,3.5759105682373047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,16,1,128,1,float16,float16,0,4.038387298583984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,16,2,128,1,float16,float16,0,4.0719856262207035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,16,2,128,1,float16,fp8,0,3.5517646789550783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,16,16,128,1,float16,fp8,0,0.5612639904022216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,16,4,128,1,float16,fp8,0,3.635567855834961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,16,4,128,1,fp8,fp8,0,3.7034961700439455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,16,4,128,1,float16,float16,0,4.210257720947266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,16,8,128,1,float16,fp8,0,3.738324737548828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,16,8,128,1,float16,float16,0,4.244513702392578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,16,128,1,float16,float16,0,2.1516416549682615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,16,8,128,1,fp8,fp8,0,3.712432098388672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,16,128,1,float16,fp8,0,1.924995231628418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,16,128,1,fp8,fp8,0,1.9306959152221679
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,16,2,128,1,fp8,fp8,0,3.6012126922607424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,1,128,1,float16,float16,0,2.0658847808837892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,1,128,1,float16,fp8,0,1.794211196899414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,1,128,1,fp8,fp8,0,1.9310047149658203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,2,128,1,float16,float16,0,2.0661760330200196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,2,128,1,float16,fp8,0,1.7978511810302735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,2,128,1,fp8,fp8,0,1.7936767578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,4,128,1,float16,float16,0,2.0469280242919923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,4,128,1,float16,fp8,0,1.8023056030273437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,4,128,1,fp8,fp8,0,1.9236064910888673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,8,128,1,float16,float16,0,1.9810752868652344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,16,128,1,float16,float16,0,1.079635238647461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,8,128,1,float16,fp8,0,1.859819221496582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,16,128,1,float16,fp8,0,0.9691967964172363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,16,8,128,1,fp8,fp8,0,1.9458032608032227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,16,128,1,fp8,fp8,0,1.09891996383667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,1,128,1,float16,fp8,0,0.9175567626953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,1,128,1,fp8,fp8,0,0.9804800033569336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,2,128,1,float16,float16,0,1.0551759719848632
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,2,128,1,float16,fp8,0,0.9033488273620606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,4,128,1,float16,float16,0,0.9719056129455567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,4,128,1,float16,fp8,0,0.9490384101867676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,4,128,1,fp8,fp8,0,0.9159952163696289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,8,128,1,float16,float16,0,1.0249024391174317
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,8,128,1,float16,fp8,0,1.0186400413513184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,16,128,1,float16,float16,0,0.5615968227386474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,8,128,1,fp8,fp8,0,0.9293600082397461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,16,128,1,float16,fp8,0,0.5029744148254395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,16,128,1,fp8,fp8,0,0.49054880142211915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,1,128,1,float16,float16,0,0.48664321899414065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,1,128,1,float16,fp8,0,0.47974238395690916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,1,128,1,fp8,fp8,0,0.5017712116241455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,2,128,1,float16,fp8,0,0.46267199516296387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,2,128,1,fp8,fp8,0,0.45480799674987793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,4,128,1,float16,float16,0,0.49244160652160646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,4,128,1,fp8,fp8,0,0.482912015914917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,8,128,1,float16,float16,0,0.5106575965881348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,8,128,1,float16,fp8,0,0.46816320419311525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,16,128,1,float16,float16,0,0.28336000442504883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,16,128,1,fp8,fp8,0,0.25281438827514646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,8,128,1,fp8,fp8,0,0.4788320064544678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,1,128,1,float16,float16,0,0.24734079837799072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,1,128,1,float16,fp8,0,0.23259999752044677
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,2,128,1,float16,float16,0,0.24895839691162108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,2,128,1,float16,fp8,0,0.23287680149078369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,2,128,1,fp8,fp8,0,0.9002464294433594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,2,128,1,fp8,fp8,0,0.23528800010681153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,4,128,1,float16,float16,0,0.25018720626831054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,4,128,1,float16,fp8,0,0.23525760173797608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,4,128,1,fp8,fp8,0,0.23687520027160644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,8,128,1,float16,float16,0,0.2584671974182129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,8,128,1,float16,fp8,0,0.24072000980377198
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,16,128,1,float16,float16,0,0.1442288041114807
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,16,128,1,float16,fp8,0,0.13262879848480225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,16,128,1,fp8,fp8,0,0.13209439516067506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,1,128,1,float16,float16,0,0.12536799907684326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,1,128,1,float16,fp8,0,0.11910079717636109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,1,128,1,fp8,fp8,0,0.11922880411148071
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,2,128,1,float16,float16,0,0.12665120363235474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,2,128,1,float16,float16,0,0.48152642250061034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,2,128,1,float16,fp8,0,0.11926720142364503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,2,128,1,fp8,fp8,0,0.11899679899215698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,4,128,1,float16,float16,0,0.13171520233154296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,4,128,1,float16,fp8,0,0.11913599967956542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,4,128,1,fp8,fp8,0,0.12016160488128662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,16,4,128,1,float16,fp8,0,0.4677855968475342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,8,128,1,fp8,fp8,0,0.12267680168151855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,16,128,1,float16,fp8,0,0.25452959537506104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,16,1,128,1,float16,float16,0,0.9636159896850586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,1,128,1,fp8,fp8,0,0.2353055953979492
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,16,8,128,1,fp8,fp8,0,0.24346721172332764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,16,1,128,1,float16,fp8,0,2.1212383270263673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,16,1,128,1,fp8,fp8,0,2.117171287536621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,16,2,128,1,float16,float16,0,2.2085264205932615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,8,128,1,float16,float16,0,0.13613120317459107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,16,2,128,1,float16,fp8,0,2.121579170227051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,16,2,128,1,fp8,fp8,0,2.1222000122070312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,16,4,128,1,float16,float16,0,2.3912303924560545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,16,4,128,1,float16,fp8,0,2.328495979309082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,16,4,128,1,fp8,fp8,0,2.1172800064086914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,16,1,128,1,float16,float16,0,2.3021823883056642
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,16,8,128,1,float16,float16,0,2.383286476135254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,16,128,1,float16,float16,0,1.3887408256530762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,16,128,1,fp8,fp8,0,1.1724687576293946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,16,8,128,1,float16,fp8,0,0.12418240308761597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,16,8,128,1,fp8,fp8,0,2.1862096786499023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,2,128,1,float16,float16,0,1.1352224349975586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,2,128,1,float16,fp8,0,1.060318374633789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,2,128,1,fp8,fp8,0,1.0678463935852052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,4,128,1,float16,float16,0,1.2047792434692384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,4,128,1,fp8,fp8,0,1.075052833557129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,16,128,1,float16,fp8,0,1.172052764892578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,1,128,1,float16,float16,0,1.1061311721801759
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,16,8,128,1,float16,fp8,0,2.202060890197754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,8,128,1,float16,fp8,0,1.1069711685180663
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,8,128,1,float16,float16,0,1.2748319625854492
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,16,128,1,float16,float16,0,0.6622367858886719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,16,128,1,float16,fp8,0,0.5932447910308838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,16,128,1,fp8,fp8,0,0.5936079978942871
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,1,128,1,float16,float16,0,0.6069007873535156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,8,128,1,fp8,fp8,0,1.1074159622192383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,1,128,1,float16,fp8,0,1.1645487785339355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,1,128,1,float16,fp8,0,0.5350080013275147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,1,128,1,fp8,fp8,0,0.5395616054534912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,2,128,1,float16,float16,0,0.56430082321167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,2,128,1,float16,fp8,0,0.5761312007904053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,2,128,1,fp8,fp8,0,0.5406879901885986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,4,128,1,float16,float16,0,0.5774352073669433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,4,128,1,float16,fp8,0,0.5402063846588134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,4,128,1,fp8,fp8,0,0.5459440231323243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,8,128,1,float16,float16,0,0.6042640209197998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,16,128,1,float16,float16,0,0.3423824071884155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,8,128,1,fp8,fp8,0,0.5605152130126954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,4,128,1,float16,fp8,0,1.0663536071777344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,16,128,1,float16,fp8,0,0.3007391929626465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,16,128,1,fp8,fp8,0,0.3038032054901123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,1,128,1,float16,float16,0,0.2906208038330078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,1,128,1,float16,fp8,0,0.2769040107727051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,1,128,1,fp8,fp8,0,0.27635040283203127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,2,128,1,float16,float16,0,0.28964800834655763
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,2,128,1,float16,fp8,0,0.2751120090484619
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,2,128,1,fp8,fp8,0,0.2776304006576538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,4,128,1,float16,float16,0,0.2993567943572998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,4,128,1,float16,fp8,0,0.28033440113067626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,4,128,1,fp8,fp8,0,0.279638409614563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,8,128,1,float16,float16,0,0.30774879455566406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,8,128,1,fp8,fp8,0,0.287225604057312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,16,128,1,float16,float16,0,0.1833951950073242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,16,128,1,fp8,fp8,0,0.15840959548950195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,1,128,1,float16,float16,0,0.14903199672698975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,1,128,1,float16,fp8,0,0.14223999977111818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,1,128,1,fp8,fp8,0,0.1440783977508545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,2,128,1,float16,float16,0,0.1499071955680847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,2,128,1,float16,fp8,0,0.14201760292053223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,2,128,1,fp8,fp8,0,0.14298720359802247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,4,128,1,float16,fp8,0,0.14462080001831054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,4,128,1,fp8,fp8,0,0.1439584016799927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,8,128,1,float16,float16,0,0.16113920211791993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,8,128,1,float16,fp8,0,0.1490048050880432
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,16,128,1,float16,float16,0,0.09901599884033203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,8,128,1,fp8,fp8,0,0.14900319576263427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,16,128,1,float16,fp8,0,0.0861407995223999
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,16,128,1,fp8,fp8,0,0.08745759725570679
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,1,128,1,float16,float16,0,0.08134880065917968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,1,128,1,float16,fp8,0,0.07888799905776978
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,16,1,128,1,fp8,fp8,0,1.1664527893066405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,2,128,1,float16,fp8,0,0.07853919863700867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,2,128,1,fp8,fp8,0,0.07857120037078857
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,4,128,1,float16,float16,0,0.08261280059814453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,4,128,1,float16,fp8,0,0.07831199765205384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,4,128,1,fp8,fp8,0,0.07869279980659485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,8,128,1,float16,float16,0,0.08948159813880921
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,8,128,1,float16,fp8,0,0.07944639921188354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,8,128,1,fp8,fp8,0,0.07947360277175904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,16,8,128,1,float16,fp8,0,0.5764336109161377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,16,8,128,1,float16,fp8,0,0.2853856086730957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,16,128,1,float16,fp8,0,0.1591472029685974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,16,4,128,1,float16,float16,0,0.15458240509033203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,16,1,128,1,float16,fp8,0,2.0885904312133787
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,16,1,128,1,fp8,fp8,0,2.083951950073242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,1,128,1,fp8,fp8,0,0.07826399803161621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,16,2,128,1,float16,float16,0,0.08152480125427246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,16,2,128,1,float16,float16,0,2.225763130187988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,16,2,128,1,float16,fp8,0,2.091200065612793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,16,2,128,1,fp8,fp8,0,2.0844240188598633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,16,4,128,1,float16,float16,0,2.2533231735229493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,16,4,128,1,float16,fp8,0,2.102948760986328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,16,4,128,1,fp8,fp8,0,2.0891439437866213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,16,8,128,1,float16,float16,0,2.406377601623535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,16,128,1,float16,float16,0,1.3241488456726074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,16,1,128,1,float16,float16,0,2.17523193359375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,16,128,1,float16,fp8,0,1.2781807899475097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,16,8,128,1,fp8,fp8,0,2.178891181945801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,16,8,128,1,float16,fp8,0,2.272097587585449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,16,128,1,fp8,fp8,0,1.1786512374877929
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,1,128,1,float16,float16,0,1.1833855628967285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,1,128,1,fp8,fp8,0,1.053774356842041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,2,128,1,float16,float16,0,1.1113871574401855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,2,128,1,float16,fp8,0,1.0414112091064454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,4,128,1,float16,float16,0,1.1228879928588866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,4,128,1,float16,fp8,0,1.1083264350891113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,4,128,1,fp8,fp8,0,1.060916805267334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,8,128,1,float16,fp8,0,1.0928112030029298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,8,128,1,float16,float16,0,1.2660335540771483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,16,128,1,float16,float16,0,0.6716879844665528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,8,128,1,fp8,fp8,0,1.1632271766662599
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,16,128,1,float16,fp8,0,0.6013279914855957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,16,128,1,fp8,fp8,0,0.5975791931152343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,1,128,1,float16,fp8,0,0.5340511798858643
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,1,128,1,fp8,fp8,0,0.5456607818603516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,2,128,1,float16,float16,0,0.5485871791839599
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,2,128,1,float16,fp8,0,0.5280655860900879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,2,128,1,fp8,fp8,0,0.5353775978088379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,4,128,1,float16,float16,0,0.5675104141235352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,4,128,1,float16,fp8,0,0.5368015766143799
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,4,128,1,fp8,fp8,0,0.5323840141296386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,1,128,1,float16,fp8,0,1.0773648262023925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,16,128,1,float16,float16,0,0.33995521068573
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,8,128,1,float16,fp8,0,0.5575871944427491
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,8,128,1,fp8,fp8,0,0.5620223999023437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,16,2,128,1,fp8,fp8,0,1.0536399841308595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,16,128,1,fp8,fp8,0,0.3084575891494751
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,1,128,1,float16,float16,0,0.2779328107833862
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,1,128,1,float16,fp8,0,0.27161440849304197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,1,128,1,fp8,fp8,0,0.27219998836517334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,2,128,1,float16,float16,0,0.28072800636291506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,2,128,1,float16,fp8,0,0.27226719856262205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,2,128,1,fp8,fp8,0,0.27310240268707275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,4,128,1,float16,float16,0,0.2907167911529541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,4,128,1,fp8,fp8,0,0.27427680492401124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,8,128,1,float16,float16,0,0.3051039934158325
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,8,128,1,float16,fp8,0,0.28641281127929685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,16,128,1,float16,float16,0,0.1768447995185852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,8,128,1,fp8,fp8,0,0.28612000942230226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,16,128,1,float16,fp8,0,0.16139839887619017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,16,128,1,fp8,fp8,0,0.16246720552444457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,1,128,1,float16,fp8,0,0.1418447971343994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,1,128,1,fp8,fp8,0,0.1421839952468872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,2,128,1,float16,float16,0,0.14677120447158815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,2,128,1,fp8,fp8,0,0.14241280555725097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,4,128,1,float16,float16,0,0.15035200119018555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,4,128,1,float16,fp8,0,0.14444960355758668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,4,128,1,fp8,fp8,0,0.14369280338287355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,8,128,1,float16,float16,0,0.15989919900894164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,8,128,1,float16,fp8,0,0.1494223952293396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,8,128,1,fp8,fp8,0,0.15004160404205322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,16,128,1,float16,fp8,0,0.08723679780960084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,16,128,1,fp8,fp8,0,0.08724160194396972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,8,128,1,float16,float16,0,0.5963007926940918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,16,128,1,float16,fp8,0,0.30598878860473633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,1,128,1,float16,float16,0,0.07552800178527833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,1,128,1,fp8,fp8,0,0.07363680005073547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,2,128,1,float16,float16,0,0.0767520010471344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,2,128,1,float16,fp8,0,0.07378720045089722
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,2,128,1,fp8,fp8,0,0.07388479709625244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,4,128,1,float16,float16,0,0.08088639974594117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,4,128,1,float16,fp8,0,0.07459999918937683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,16,4,128,1,float16,fp8,0,0.27538719177246096
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,8,128,1,float16,float16,0,0.08539999723434448
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,8,128,1,float16,fp8,0,0.07872800230979919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,8,128,1,fp8,fp8,0,0.07841920256614685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,16,128,1,float16,float16,0,0.051316797733306885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,16,128,1,float16,fp8,0,0.045988801121711734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,16,128,1,fp8,fp8,0,0.04543519914150238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,1,128,1,float16,float16,0,0.14426079988479615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,1,128,1,float16,fp8,0,0.04158399999141693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,1,128,1,fp8,fp8,0,0.04110080003738403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,2,128,1,float16,float16,0,0.042247998714447024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,16,2,128,1,float16,fp8,0,0.1420240044593811
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,2,128,1,fp8,fp8,0,0.04121119976043701
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,4,128,1,float16,float16,0,0.042587199807167055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,4,128,1,float16,fp8,0,0.0416047990322113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,4,128,1,fp8,fp8,0,0.041950398683547975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,8,128,1,float16,float16,0,0.044207999110221864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,16,1,128,1,float16,float16,0,0.6050496101379395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,8,128,1,fp8,fp8,0,0.042633599042892455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,16,128,1,float16,float16,0,0.09375839829444885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,16,1,128,1,float16,float16,0,1.3172543525695801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,1,128,1,float16,fp8,0,0.07332959771156311
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,16,1,128,1,fp8,fp8,0,1.2891119956970214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,16,4,128,1,fp8,fp8,0,0.07365919947624207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,1,128,1,float16,float16,0,0.041812801361083986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,16,2,128,1,float16,float16,0,1.4273823738098144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,16,2,128,1,float16,fp8,0,1.2931952476501465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,2,128,1,float16,fp8,0,0.04139359891414642
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,16,2,128,1,fp8,fp8,0,1.2902159690856934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,16,8,128,1,float16,fp8,0,0.042105600237846375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,16,4,128,1,float16,float16,0,1.362987232208252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,16,4,128,1,float16,fp8,0,1.3073648452758788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,16,4,128,1,fp8,fp8,0,1.3836544036865235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,16,8,128,1,float16,fp8,0,1.364742374420166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,16,128,1,float16,float16,0,0.8345935821533204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,16,8,128,1,fp8,fp8,0,1.3702447891235352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,16,1,128,1,float16,fp8,0,1.3001184463500977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,16,128,1,float16,fp8,0,0.7590832233428955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,1,128,1,float16,float16,0,0.6564383983612061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,1,128,1,float16,fp8,0,0.6561535835266114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,1,128,1,fp8,fp8,0,0.6509823799133301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,2,128,1,float16,fp8,0,0.6536511898040771
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,2,128,1,fp8,fp8,0,0.6484784126281739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,4,128,1,float16,float16,0,0.6918511867523194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,4,128,1,float16,fp8,0,0.6597583770751954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,4,128,1,fp8,fp8,0,0.6614943981170655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,8,128,1,float16,float16,0,0.7357903957366944
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,8,128,1,float16,fp8,0,0.7248479843139648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,16,128,1,float16,float16,0,0.42513117790222166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,8,128,1,fp8,fp8,0,0.6935823917388916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,16,128,1,float16,fp8,0,0.38471999168396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,16,128,1,fp8,fp8,0,0.3946239948272705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,1,128,1,float16,float16,0,0.33622400760650634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,1,128,1,float16,fp8,0,0.33287360668182375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,1,128,1,fp8,fp8,0,0.3308703899383545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,16,8,128,1,float16,float16,0,1.479478359222412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,2,128,1,fp8,fp8,0,0.33368959426879885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,4,128,1,float16,float16,0,0.3496432065963745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,4,128,1,float16,fp8,0,0.34750559329986574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,8,128,1,float16,float16,0,0.37435359954833985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,16,128,1,fp8,fp8,0,0.7940239906311035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,8,128,1,float16,fp8,0,0.36448159217834475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,16,2,128,1,float16,float16,0,0.6652863979339599
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,16,128,1,float16,float16,0,0.22444961071014405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,8,128,1,fp8,fp8,0,0.35150880813598634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,16,128,1,float16,fp8,0,0.20271039009094238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,16,128,1,fp8,fp8,0,0.2015984058380127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,1,128,1,fp8,fp8,0,0.17505279779434205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,2,128,1,float16,float16,0,0.17747199535369873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,2,128,1,float16,fp8,0,0.1753424048423767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,2,128,1,fp8,fp8,0,0.17407360076904296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,4,128,1,float16,float16,0,0.18456159830093383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,4,128,1,float16,fp8,0,0.17610080242156984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,4,128,1,fp8,fp8,0,0.17498559951782228
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,8,128,1,float16,float16,0,0.19676480293273926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,8,128,1,float16,fp8,0,0.18515360355377197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,8,128,1,fp8,fp8,0,0.18434079885482788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,16,128,1,float16,float16,0,0.11873120069503784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,16,128,1,float16,fp8,0,0.10894880294799805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,16,128,1,fp8,fp8,0,0.10815520286560058
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,1,128,1,float16,float16,0,0.09432640075683593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,1,128,1,float16,fp8,0,0.0912880003452301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,1,128,1,fp8,fp8,0,0.09000639915466309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,2,128,1,float16,float16,0,0.09613440036773682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,2,128,1,float16,fp8,0,0.09201120138168335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,2,128,1,fp8,fp8,0,0.09133759737014771
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,4,128,1,float16,float16,0,0.09860159754753113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,4,128,1,float16,fp8,0,0.09397600293159485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,2,128,1,float16,fp8,0,0.3340015888214111
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,4,128,1,fp8,fp8,0,0.093094402551651
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,8,128,1,float16,float16,0,0.10480799674987792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,8,128,1,float16,fp8,0,0.09964320063591003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,16,8,128,1,fp8,fp8,0,0.09971519708633422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,16,128,1,float16,fp8,0,0.05919359922409058
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,16,128,1,fp8,fp8,0,0.0586031973361969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,1,128,1,float16,float16,0,0.05023840069770813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,1,128,1,float16,fp8,0,0.04998559951782226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,1,128,1,fp8,fp8,0,0.04963200092315674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,4,128,1,fp8,fp8,0,0.3381999969482422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,2,128,1,float16,fp8,0,0.050070399045944215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,2,128,1,fp8,fp8,0,0.0494704008102417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,4,128,1,float16,float16,0,0.05168480277061462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,4,128,1,float16,fp8,0,0.05011039972305298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,4,128,1,fp8,fp8,0,0.049686399102210996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,8,128,1,float16,float16,0,0.05833439826965332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,8,128,1,float16,fp8,0,0.05304960012435913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,16,128,1,float16,float16,0,0.034153598546981814
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,16,128,1,float16,fp8,0,0.03244799971580505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,16,128,1,fp8,fp8,0,0.032183998823165895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,1,128,1,float16,float16,0,0.030723199248313904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,1,128,1,float16,fp8,0,0.029998400807380678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,1,128,1,fp8,fp8,0,0.030700799822807313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,1,128,1,float16,float16,0,0.17698400020599364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,2,128,1,float16,fp8,0,0.0305184006690979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,2,128,1,fp8,fp8,0,0.030641600489616394
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,4,128,1,float16,float16,0,0.031307199597358705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,4,128,1,float16,fp8,0,0.03091680109500885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,8,128,1,float16,float16,0,0.032507199048995974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,8,128,1,float16,fp8,0,0.03130399882793426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,8,128,1,fp8,fp8,0,0.0313264012336731
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,16,2,128,1,float16,float16,0,0.34651360511779783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,16,1,128,1,float16,float16,0,1.341198444366455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,16,128,1,float16,float16,0,0.0652288019657135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,2,128,1,float16,float16,0,0.05025280117988586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,16,8,128,1,fp8,fp8,0,0.052744001150131226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,16,1,128,1,float16,fp8,0,0.1716480016708374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,16,1,128,1,fp8,fp8,0,1.349283218383789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,16,2,128,1,float16,float16,0,1.3517295837402343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,16,2,128,1,float16,fp8,0,1.3461039543151856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,16,2,128,1,fp8,fp8,0,1.3430272102355958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,16,4,128,1,float16,float16,0,1.4172160148620605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,16,4,128,1,float16,fp8,0,1.3640080451965333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,16,4,128,1,fp8,fp8,0,1.3607312202453614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,16,8,128,1,float16,float16,0,1.5469216346740722
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,16,128,1,float16,float16,0,0.917244815826416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,16,8,128,1,float16,fp8,0,1.4816576004028321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,16,8,128,1,fp8,fp8,0,1.4530431747436523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,16,1,128,1,float16,fp8,0,1.3486047744750977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,4,128,1,fp8,fp8,0,0.03091999888420105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,16,128,1,fp8,fp8,0,0.8213727951049805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,1,128,1,float16,float16,0,0.6716479778289794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,1,128,1,float16,fp8,0,0.6784848213195801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,1,128,1,fp8,fp8,0,0.678607988357544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,2,128,1,float16,float16,0,0.6880703926086426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,2,128,1,float16,fp8,0,0.6776591777801514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,2,128,1,fp8,fp8,0,0.6784416198730469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,4,128,1,float16,float16,0,0.7175151824951171
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,4,128,1,float16,fp8,0,0.6889200210571289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,4,128,1,fp8,fp8,0,0.6905392169952392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,8,128,1,float16,float16,0,0.7801023960113526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,8,128,1,float16,fp8,0,0.7322368144989013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,16,128,1,float16,float16,0,0.46471362113952636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,8,128,1,fp8,fp8,0,0.7558015823364258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,16,128,1,float16,fp8,0,0.419049596786499
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,16,128,1,fp8,fp8,0,0.41809759140014646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,1,128,1,float16,fp8,0,0.3452368021011353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,1,128,1,fp8,fp8,0,0.3527616024017334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,2,128,1,float16,float16,0,0.3513871908187866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,2,128,1,float16,fp8,0,0.34622879028320314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,2,128,1,fp8,fp8,0,0.346014404296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,16,16,128,1,float16,fp8,0,0.842739200592041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,4,128,1,float16,float16,0,0.3679487943649292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,4,128,1,float16,fp8,0,0.35267200469970705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,4,128,1,fp8,fp8,0,0.35220000743865965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,8,128,1,float16,float16,0,0.39767520427703856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,8,128,1,fp8,fp8,0,0.3766319990158081
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,8,128,1,float16,fp8,0,0.3757472038269043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,16,128,1,float16,fp8,0,0.21597440242767335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,16,128,1,fp8,fp8,0,0.21590399742126465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,1,128,1,float16,fp8,0,0.1800528049468994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,1,128,1,fp8,fp8,0,0.17974720001220704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,2,128,1,float16,float16,0,0.18171679973602295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,2,128,1,fp8,fp8,0,0.18059840202331542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,2,128,1,float16,fp8,0,0.17972320318222046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,4,128,1,float16,float16,0,0.19070559740066528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,4,128,1,float16,fp8,0,0.18234879970550538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,8,128,1,float16,float16,0,0.20610721111297609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,8,128,1,float16,fp8,0,0.19403840303421022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,8,128,1,fp8,fp8,0,0.19302719831466675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,16,128,1,float16,float16,0,0.1272752046585083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,16,128,1,float16,fp8,0,0.11501760482788086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,16,128,1,fp8,fp8,0,0.11470400094985962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,1,128,1,float16,fp8,0,0.09613119959831237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,1,128,1,fp8,fp8,0,0.09570559859275818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,2,128,1,float16,float16,0,0.09769920110702515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,2,128,1,float16,fp8,0,0.09600639939308167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,2,128,1,fp8,fp8,0,0.09559040069580078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,4,128,1,float16,float16,0,0.10142719745635986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,4,128,1,float16,fp8,0,0.09732800126075744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,4,128,1,fp8,fp8,0,0.09778239727020263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,8,128,1,float16,float16,0,0.11005760431289673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,8,128,1,float16,fp8,0,0.10368959903717041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,8,128,1,fp8,fp8,0,0.1033087968826294
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,16,128,1,float16,float16,0,0.06907200217247009
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,16,128,1,float16,fp8,0,0.06224160194396973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,16,128,1,fp8,fp8,0,0.0627232015132904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,1,128,1,float16,float16,0,0.049542400240898135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,1,128,1,float16,fp8,0,0.05108640193939209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,1,128,1,fp8,fp8,0,0.05079200267791748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,16,2,128,1,float16,float16,0,0.03047040104866028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,2,128,1,float16,float16,0,0.05216320157051087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,2,128,1,fp8,fp8,0,0.050937598943710326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,4,128,1,float16,float16,0,0.05618559718132019
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,16,128,1,float16,float16,0,0.24107999801635743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,4,128,1,fp8,fp8,0,0.050990402698516846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,8,128,1,float16,float16,0,0.06038879752159119
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,1,128,1,float16,float16,0,0.1795583963394165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,8,128,1,fp8,fp8,0,0.05493119955062866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,16,128,1,float16,float16,0,0.03865599930286408
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,16,128,1,float16,fp8,0,0.03236800134181976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,16,128,1,fp8,fp8,0,0.03285439908504486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,1,128,1,float16,float16,0,0.029120001196861266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,1,128,1,float16,fp8,0,0.030078399181365966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,1,128,1,fp8,fp8,0,0.030140799283981324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,2,128,1,float16,float16,0,0.029572799801826477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,16,4,128,1,fp8,fp8,0,0.183515202999115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,2,128,1,float16,fp8,0,0.0301503986120224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,2,128,1,fp8,fp8,0,0.0301503986120224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,4,128,1,float16,float16,0,0.030028799176216127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,4,128,1,fp8,fp8,0,0.030350399017333985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,8,128,1,float16,float16,0,0.031672000885009766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,8,128,1,float16,fp8,0,0.031119999289512635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,8,128,1,fp8,fp8,0,0.03146879971027374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,16,128,1,float16,float16,0,0.026649600267410277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,16,1,128,1,float16,float16,0,0.09589120149612426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,16,128,1,float16,fp8,0,0.026291200518608095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,1,128,1,float16,float16,0,0.02401120066642761
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,1,128,1,float16,fp8,0,0.02499680072069168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,1,128,1,fp8,fp8,0,0.02466239929199219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,2,128,1,float16,float16,0,0.024291199445724488
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,2,128,1,float16,fp8,0,0.024542400240898134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,2,128,1,fp8,fp8,0,0.02470880001783371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,4,128,1,float16,float16,0,0.02417919933795929
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,4,128,1,float16,fp8,0,0.02480800002813339
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,16,1,128,1,float16,float16,0,0.34331040382385253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,4,128,1,fp8,fp8,0,0.024537600576877594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,8,128,1,float16,float16,0,0.025128000974655153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,8,128,1,fp8,fp8,0,0.026099199056625368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,2,128,1,float16,fp8,0,0.05103999972343445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,4,128,1,float16,fp8,0,0.050692802667617796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,16,1,128,1,float16,float16,0,0.9383600234985352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,16,8,128,1,float16,fp8,0,0.055232000350952146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,16,1,128,1,float16,fp8,0,0.970036792755127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,16,4,128,1,float16,fp8,0,0.030353599786758424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,16,1,128,1,fp8,fp8,0,0.9766863822937012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,16,2,128,1,float16,float16,0,0.9620880126953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,16,2,128,1,float16,fp8,0,0.9793951988220215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,16,128,1,fp8,fp8,0,0.02656799852848053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,16,2,128,1,fp8,fp8,0,0.9732928276062012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,16,4,128,1,float16,float16,0,1.0289199829101563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,16,4,128,1,float16,fp8,0,1.0015999794006347
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,16,4,128,1,fp8,fp8,0,0.9958352088928223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,16,8,128,1,float16,float16,0,1.1598560333251953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,16,8,128,1,float16,fp8,0,1.1054271697998046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,16,128,1,float16,float16,0,0.7203504085540772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,16,128,1,float16,fp8,0,0.6547920227050781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,16,128,1,fp8,fp8,0,0.6512735843658447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,16,8,128,1,fp8,fp8,0,1.090998363494873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,1,128,1,float16,float16,0,0.48055357933044435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,1,128,1,float16,fp8,0,0.4930704116821289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,2,128,1,float16,float16,0,0.49010400772094725
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,2,128,1,float16,fp8,0,0.49443039894104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,2,128,1,fp8,fp8,0,0.4939551830291748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,4,128,1,float16,fp8,0,0.5056560039520264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,4,128,1,fp8,fp8,0,0.5079872131347656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,8,128,1,float16,float16,0,0.5843023777008056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,8,128,1,float16,fp8,0,0.5534255981445313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,8,128,1,fp8,fp8,0,0.5540719985961914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,16,128,1,float16,float16,0,0.36798079013824464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,16,128,1,float16,fp8,0,0.33326880931854247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,1,128,1,float16,float16,0,0.24660320281982423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,1,128,1,float16,fp8,0,0.25430080890655515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,2,128,1,float16,float16,0,0.2507359981536865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,2,128,1,float16,fp8,0,0.25478079319000246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,2,128,1,fp8,fp8,0,0.254800009727478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,4,128,1,float16,float16,0,0.2665055990219116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,16,8,128,1,float16,fp8,0,0.024872000515460967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,4,128,1,float16,fp8,0,0.2601759910583496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,4,128,1,fp8,fp8,0,0.2609472036361694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,8,128,1,float16,float16,0,0.2988384008407593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,8,128,1,float16,fp8,0,0.28588318824768066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,8,128,1,fp8,fp8,0,0.2839855909347534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,16,128,1,float16,float16,0,0.19071840047836303
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,16,128,1,fp8,fp8,0,0.17367360591888428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,1,128,1,float16,fp8,0,0.13371360301971436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,1,128,1,fp8,fp8,0,0.13374719619750977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,2,128,1,float16,float16,0,0.13387199640274047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,2,128,1,float16,fp8,0,0.134280002117157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,2,128,1,fp8,fp8,0,0.1346287965774536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,1,128,1,fp8,fp8,0,0.49532642364501955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,4,128,1,float16,float16,0,0.14147839546203614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,4,128,1,float16,fp8,0,0.1379040002822876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,4,128,1,fp8,fp8,0,0.13763359785079957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,8,128,1,float16,float16,0,0.15715839862823486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,8,128,1,float16,fp8,0,0.14952319860458374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,16,128,1,float16,float16,0,0.10256799459457397
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,16,128,1,float16,fp8,0,0.0936847984790802
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,16,128,1,fp8,fp8,0,0.09397119879722596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,1,128,1,float16,float16,0,0.07143359780311584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,1,128,1,float16,fp8,0,0.07347040176391602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,1,128,1,fp8,fp8,0,0.07324320077896118
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,2,128,1,float16,float16,0,0.07345439791679383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,2,128,1,fp8,fp8,0,0.07369920015335082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,16,128,1,fp8,fp8,0,0.3328480005264282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,16,1,128,1,fp8,fp8,0,0.25497119426727294
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,4,128,1,float16,fp8,0,0.0751695990562439
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,4,128,1,fp8,fp8,0,0.07510079741477967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,8,128,1,float16,float16,0,0.08549919724464417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,8,128,1,float16,fp8,0,0.08149759769439698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,8,128,1,fp8,fp8,0,0.08113279938697815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,16,128,1,float16,float16,0,0.05825759768486023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,16,128,1,float16,fp8,0,0.0542639970779419
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,16,128,1,fp8,fp8,0,0.05419359803199768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,1,128,1,float16,fp8,0,0.040428799390792844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,1,128,1,fp8,fp8,0,0.040715199708938596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,2,128,1,float16,float16,0,0.04122720062732697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,1,128,1,float16,float16,0,0.12968480587005615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,2,128,1,float16,fp8,0,0.04063679873943329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,2,128,1,fp8,fp8,0,0.040300801396369934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,4,128,1,float16,float16,0,0.04532159864902496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,4,128,1,float16,fp8,0,0.040566399693489075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,4,128,1,fp8,fp8,0,0.040884798765182494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,8,128,1,float16,float16,0,0.05012480020523071
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,8,128,1,float16,fp8,0,0.04473280012607574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,8,128,1,fp8,fp8,0,0.04541760087013245
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,16,4,128,1,float16,float16,0,0.5247807979583741
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,16,128,1,float16,fp8,0,0.03035680055618286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,1,128,1,float16,float16,0,0.025271999835968017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,16,128,1,fp8,fp8,0,0.030755200982093812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,1,128,1,float16,fp8,0,0.0262719988822937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,8,128,1,fp8,fp8,0,0.15003999471664428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,2,128,1,float16,float16,0,0.025889599323272706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,2,128,1,float16,fp8,0,0.026443201303482055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,2,128,1,fp8,fp8,0,0.026358398795127868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,4,128,1,float16,float16,0,0.026335999369621277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,2,128,1,float16,fp8,0,0.07376800179481506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,4,128,1,float16,fp8,0,0.026655998826026917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,4,128,1,fp8,fp8,0,0.026440000534057616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,8,128,1,float16,fp8,0,0.027263998985290527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,16,4,128,1,float16,float16,0,0.07766079902648926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,8,128,1,fp8,fp8,0,0.02725600004196167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,16,128,1,float16,float16,0,0.019784000515937806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,16,128,1,float16,fp8,0,0.02030239999294281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,16,128,1,fp8,fp8,0,0.020260800421237946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,1,128,1,float16,float16,0,0.017484800517559053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,1,128,1,float16,fp8,0,0.018852800130844116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,1,128,1,fp8,fp8,0,0.01886560022830963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,2,128,1,float16,fp8,0,0.018892799317836762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,16,1,128,1,float16,float16,0,0.03993119895458221
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,4,128,1,float16,float16,0,0.017873600125312805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,4,128,1,float16,fp8,0,0.01920959949493408
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,16,16,128,1,float16,fp8,0,0.17361600399017335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,4,128,1,fp8,fp8,0,0.019337600469589232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,8,128,1,float16,float16,0,0.018777599930763243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,8,128,1,fp8,fp8,0,0.01958879977464676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,16,128,1,float16,float16,0,0.01865759938955307
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,16,128,1,float16,fp8,0,0.019659200310707094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,16,128,1,fp8,fp8,0,0.019681599736213685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,1,128,1,float16,float16,0,0.017399999499320983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,1,128,1,float16,fp8,0,0.018913599848747253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,16,128,1,float16,float16,0,0.034332799911499026
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,2,128,1,float16,float16,0,0.017768000066280366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,2,128,1,float16,fp8,0,0.018913599848747253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,1,128,1,fp8,fp8,0,0.026417601108551025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,2,128,1,fp8,fp8,0,0.018961599469184874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,4,128,1,float16,float16,0,0.01762239933013916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,4,128,1,float16,fp8,0,0.0188960000872612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,4,128,1,fp8,fp8,0,0.0188511997461319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,8,128,1,float16,float16,0,0.017953599989414214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,8,128,1,float16,fp8,0,0.019152000546455383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,8,128,1,fp8,fp8,0,0.019337600469589232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,16,8,128,1,float16,float16,0,0.027849599719047546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,16,1,128,1,float16,float16,0,0.3906016111373901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,16,1,128,1,float16,fp8,0,0.41178078651428224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,16,1,128,1,fp8,fp8,0,0.4115039825439453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,16,2,128,1,float16,float16,0,0.40183358192443847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,16,2,128,1,float16,fp8,0,0.4135312080383301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,2,128,1,fp8,fp8,0,0.01903519928455353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,16,2,128,1,fp8,fp8,0,0.412992000579834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,16,4,128,1,float16,float16,0,0.43265438079833984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,16,4,128,1,float16,fp8,0,0.4251920223236084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,16,4,128,1,fp8,fp8,0,0.42440481185913087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,16,8,128,1,float16,float16,0,0.49672799110412597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,16,8,128,1,float16,fp8,0,0.4757088184356689
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,16,128,1,float16,float16,0,0.3241136074066162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,16,8,128,1,fp8,fp8,0,0.47419037818908694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,16,128,1,float16,fp8,0,0.27848799228668214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,16,1,128,1,fp8,fp8,0,0.018756799399852753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,1,128,1,float16,float16,0,0.2011888027191162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,1,128,1,float16,fp8,0,0.21309759616851806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,1,128,1,fp8,fp8,0,0.21288959980010985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,2,128,1,float16,float16,0,0.20610399246215821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,2,128,1,fp8,fp8,0,0.21338241100311278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,4,128,1,float16,float16,0,0.2229856014251709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,4,128,1,float16,fp8,0,0.22033119201660156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,4,128,1,fp8,fp8,0,0.21816799640655518
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,8,128,1,float16,float16,0,0.25476961135864257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,8,128,1,float16,fp8,0,0.2446687936782837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,8,128,1,fp8,fp8,0,0.24444639682769775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,16,128,1,fp8,fp8,0,0.1467695951461792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,1,128,1,float16,float16,0,0.10815680027008057
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,1,128,1,float16,fp8,0,0.11369760036468506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,1,128,1,fp8,fp8,0,0.11314879655838013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,2,128,1,float16,float16,0,0.11132160425186158
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,2,128,1,float16,fp8,0,0.11358879804611206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,2,128,1,float16,float16,0,0.01780800074338913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,4,128,1,float16,float16,0,0.11890079975128173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,4,128,1,float16,fp8,0,0.11628639698028564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,4,128,1,fp8,fp8,0,0.11680959463119507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,8,128,1,float16,float16,0,0.13478560447692872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,8,128,1,float16,fp8,0,0.12922559976577758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,8,128,1,fp8,fp8,0,0.12929439544677734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,16,128,1,float16,float16,0,0.09252480268478394
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,16,128,1,float16,fp8,0,0.0807424008846283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,16,128,1,fp8,fp8,0,0.08059200048446655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,1,128,1,float16,float16,0,0.06088160276412964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,1,128,1,float16,fp8,0,0.06281440258026123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,2,128,1,float16,float16,0,0.06269279718399048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,16,128,1,fp8,fp8,0,0.27939040660858155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,2,128,1,float16,fp8,0,0.0632207989692688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,2,128,1,fp8,fp8,0,0.06321600079536438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,4,128,1,float16,float16,0,0.06684799790382386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,4,128,1,float16,fp8,0,0.06483359932899475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,4,128,1,fp8,fp8,0,0.06483359932899475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,8,128,1,float16,float16,0,0.07493600249290466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,8,128,1,float16,fp8,0,0.0718608021736145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,8,128,1,fp8,fp8,0,0.07148320078849793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,16,128,1,float16,float16,0,0.05294560194015503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,16,128,1,float16,fp8,0,0.04624800086021423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,16,128,1,fp8,fp8,0,0.046300798654556274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,1,128,1,float16,float16,0,0.03323520123958588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,16,128,1,float16,fp8,0,0.14652320146560668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,1,128,1,float16,fp8,0,0.035097599029541016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,1,128,1,fp8,fp8,0,0.03510879874229431
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,2,128,1,float16,float16,0,0.03478559851646423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,2,128,1,float16,fp8,0,0.03503200113773346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,2,128,1,fp8,fp8,0,0.035087999701499936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,4,128,1,float16,float16,0,0.03840959966182709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,4,128,1,float16,fp8,0,0.035604798793792726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,4,128,1,fp8,fp8,0,0.03525919914245605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,8,128,1,float16,float16,0,0.04373280107975006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,8,128,1,float16,fp8,0,0.039083200693130496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,2,128,1,fp8,fp8,0,0.11380319595336914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,16,8,128,1,fp8,fp8,0,0.03931359946727753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,16,128,1,float16,float16,0,0.030955201387405394
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,16,128,1,float16,fp8,0,0.025313600897789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,16,128,1,fp8,fp8,0,0.02510559856891632
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,1,128,1,float16,float16,0,0.021083199977874757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,1,128,1,float16,fp8,0,0.022809599339962006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,1,128,1,fp8,fp8,0,0.02253440022468567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,2,128,1,float16,float16,0,0.021379199624061585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,2,128,1,float16,fp8,0,0.022707200050354003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,2,128,1,fp8,fp8,0,0.02258239984512329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,4,128,1,float16,fp8,0,0.022888000309467315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,4,128,1,fp8,fp8,0,0.022784000635147093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,8,128,1,float16,float16,0,0.02321600019931793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,8,128,1,float16,fp8,0,0.0235615998506546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,8,128,1,fp8,fp8,0,0.023902399837970732
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,16,128,1,float16,fp8,0,0.01668799966573715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,16,128,1,fp8,fp8,0,0.01671999990940094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,1,128,1,float16,float16,0,0.014820800721645355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,1,128,1,float16,fp8,0,0.015468800067901611
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,16,1,128,1,fp8,fp8,0,0.06262720227241517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,1,128,1,fp8,fp8,0,0.015352000296115876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,2,128,1,float16,float16,0,0.01478399932384491
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,2,128,1,fp8,fp8,0,0.015279999375343323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,4,128,1,float16,float16,0,0.015081599354743958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,4,128,1,float16,fp8,0,0.015329599380493164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,4,128,1,fp8,fp8,0,0.01533920019865036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,16,2,128,1,float16,fp8,0,0.2121583938598633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,8,128,1,float16,float16,0,0.015779200196266174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,8,128,1,float16,fp8,0,0.016209599375724793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,16,128,1,float16,float16,0,0.015694400668144225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,16,128,1,float16,fp8,0,0.01592320054769516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,16,128,1,fp8,fp8,0,0.016014400124549865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,16,16,128,1,float16,float16,0,0.16965119838714598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,1,128,1,float16,float16,0,0.014524799585342408
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,1,128,1,float16,fp8,0,0.014975999295711518
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,2,128,1,float16,float16,0,0.01438560038805008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,2,128,1,float16,fp8,0,0.015240000188350677
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,2,128,1,fp8,fp8,0,0.015132799744606018
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,4,128,1,float16,float16,0,0.0146479994058609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,4,128,1,float16,fp8,0,0.01523360013961792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,4,128,1,fp8,fp8,0,0.0150736004114151
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,8,128,1,float16,float16,0,0.014833599328994751
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,8,128,1,float16,fp8,0,0.015427200496196747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,16,4,128,1,float16,float16,0,0.02181120067834854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,8,128,1,fp8,fp8,0,0.01539199948310852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,16,128,1,float16,float16,0,0.014734399318695069
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,16,8,128,1,float16,fp8,0,0.019729599356651306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,16,128,1,float16,float16,0,0.017070400714874267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,1,128,1,float16,float16,0,0.014001600444316864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,1,128,1,float16,fp8,0,0.014963200688362122
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,2,128,1,float16,float16,0,0.01414559930562973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,2,128,1,float16,fp8,0,0.014833599328994751
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,2,128,1,fp8,fp8,0,0.014881600439548493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,4,128,1,float16,float16,0,0.013993600010871887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,2,128,1,float16,fp8,0,0.015352000296115876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,4,128,1,fp8,fp8,0,0.014918400347232819
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,8,128,1,float16,float16,0,0.014416000247001648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,8,128,1,float16,fp8,0,0.015337599813938141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,8,128,1,fp8,fp8,0,0.01536799967288971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,16,8,128,1,fp8,fp8,0,0.015859200060367583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,16,1,128,1,float16,float16,0,0.1881760001182556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,16,1,128,1,fp8,fp8,0,0.20546560287475585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,16,2,128,1,float16,float16,0,0.19410400390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,16,1,128,1,fp8,fp8,0,0.015248000621795654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,16,2,128,1,float16,fp8,0,0.20605120658874512
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,16,2,128,1,fp8,fp8,0,0.2030656099319458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,16,4,128,1,float16,float16,0,0.2087951898574829
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,16,4,128,1,float16,fp8,0,0.21747200489044188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,16,4,128,1,fp8,fp8,0,0.21506879329681397
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,16,8,128,1,float16,float16,0,0.24444479942321778
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,16,8,128,1,float16,fp8,0,0.23702559471130372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,16,8,128,1,fp8,fp8,0,0.2363055944442749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,16,128,1,float16,float16,0,0.16949440240859986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,16,128,1,float16,fp8,0,0.14458240270614625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,16,128,1,float16,fp8,0,0.015364800393581391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,16,128,1,fp8,fp8,0,0.0152879998087883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,1,128,1,fp8,fp8,0,0.10668799877166749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,2,128,1,float16,float16,0,0.10348800420761109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,2,128,1,float16,fp8,0,0.107806396484375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,2,128,1,fp8,fp8,0,0.10858559608459473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,4,128,1,float16,float16,0,0.11027040481567382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,4,128,1,float16,fp8,0,0.11227680444717407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,4,128,1,float16,fp8,0,0.01494079977273941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,8,128,1,float16,float16,0,0.13027679920196533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,8,128,1,float16,fp8,0,0.12524800300598143
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,8,128,1,fp8,fp8,0,0.12504639625549316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,16,128,1,float16,float16,0,0.09148640036582947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,16,128,1,float16,fp8,0,0.07755839824676514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,16,128,1,fp8,fp8,0,0.07753120064735412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,1,128,1,float16,float16,0,0.05751519799232483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,1,128,1,float16,fp8,0,0.058462399244308474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,1,128,1,fp8,fp8,0,0.05862879753112793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,2,128,1,float16,float16,0,0.05947039723396301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,2,128,1,float16,fp8,0,0.05923519730567932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,2,128,1,fp8,fp8,0,0.05916320085525513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,4,128,1,float16,float16,0,0.06370559930801392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,4,128,1,float16,fp8,0,0.06158879995346069
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,4,128,1,fp8,fp8,0,0.061312001943588254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,16,128,1,fp8,fp8,0,0.14415199756622316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,8,128,1,float16,float16,0,0.0716480016708374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,8,128,1,float16,fp8,0,0.06677119731903076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,16,8,128,1,fp8,fp8,0,0.06665120124816895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,16,1,128,1,fp8,fp8,0,0.014953599870204925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,16,128,1,float16,float16,0,0.05028319954872131
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,16,128,1,float16,fp8,0,0.04164479970932007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,16,128,1,fp8,fp8,0,0.04176000058650971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,1,128,1,float16,fp8,0,0.030351999402046203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,1,128,1,fp8,fp8,0,0.030374398827552794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,2,128,1,float16,float16,0,0.030151998996734618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,2,128,1,float16,fp8,0,0.030369600653648375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,4,128,1,fp8,fp8,0,0.11170719861984253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,4,128,1,float16,float16,0,0.03569599986076355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,4,128,1,float16,fp8,0,0.0310479998588562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,4,128,1,fp8,fp8,0,0.030660799145698546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,8,128,1,float16,float16,0,0.041254401206970215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,8,128,1,float16,fp8,0,0.03376320004463196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,8,128,1,fp8,fp8,0,0.034283199906349184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,16,128,1,float16,float16,0,0.028911998867988585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,16,128,1,float16,fp8,0,0.020817600190639496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,16,128,1,fp8,fp8,0,0.021147200465202333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,1,128,1,float16,float16,0,0.01778080016374588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,1,128,1,float16,fp8,0,0.019307200610637665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,1,128,1,fp8,fp8,0,0.018886399269104005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,2,128,1,float16,float16,0,0.018718400597572328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,2,128,1,float16,fp8,0,0.019012799859046935
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,2,128,1,fp8,fp8,0,0.019355200231075287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,4,128,1,float16,float16,0,0.01889120042324066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,4,128,1,float16,fp8,0,0.019340799748897554
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,4,128,1,fp8,fp8,0,0.01945440024137497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,8,128,1,float16,float16,0,0.020707200467586517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,8,128,1,float16,fp8,0,0.019916799664497376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,16,8,128,1,fp8,fp8,0,0.019812799990177155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,16,128,1,float16,float16,0,0.015315200388431548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,16,128,1,float16,fp8,0,0.014556799829006196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,16,128,1,fp8,fp8,0,0.014643199741840363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,1,128,1,float16,float16,0,0.012862400710582733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,1,128,1,float16,fp8,0,0.013708800077438354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,1,128,1,fp8,fp8,0,0.013763199746608733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,2,128,1,float16,float16,0,0.013502399623394012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,2,128,1,fp8,fp8,0,0.013979199528694152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,4,128,1,float16,float16,0,0.013711999356746673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,4,128,1,float16,fp8,0,0.014071999490261078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,4,128,1,fp8,fp8,0,0.014052799344062806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,8,128,1,float16,float16,0,0.014262400567531586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,8,128,1,float16,fp8,0,0.014139199256896972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,8,128,1,fp8,fp8,0,0.014163200557231904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,1,128,1,float16,fp8,0,0.10794559717178345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,16,128,1,float16,float16,0,0.013828800618648529
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,16,128,1,float16,fp8,0,0.013801600039005279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,16,128,1,fp8,fp8,0,0.013704000413417817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,1,128,1,float16,float16,0,0.012809599936008453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,1,128,1,float16,fp8,0,0.013324800133705138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,1,128,1,fp8,fp8,0,0.013451200723648072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,2,128,1,float16,float16,0,0.012886400520801543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,2,128,1,float16,fp8,0,0.013358399271965027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,2,128,1,fp8,fp8,0,0.013558399677276612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,4,128,1,float16,float16,0,0.012895999848842621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,4,128,1,float16,fp8,0,0.013542400300502777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,4,128,1,fp8,fp8,0,0.013340799510478974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,8,128,1,float16,float16,0,0.013300800323486328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,2,128,1,fp8,fp8,0,0.030633598566055298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,8,128,1,float16,fp8,0,0.013395200669765472
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,16,8,128,1,fp8,fp8,0,0.013644799590110779
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,16,128,1,float16,float16,0,0.013176000118255616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,16,128,1,fp8,fp8,0,0.013150399923324585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,1,128,1,float16,fp8,0,0.013288000226020813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,1,128,1,fp8,fp8,0,0.013012799620628356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,2,128,1,float16,float16,0,0.012593600153923034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,2,128,1,float16,fp8,0,0.013115200400352477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,2,128,1,fp8,fp8,0,0.013438400626182557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,4,128,1,float16,float16,0,0.012744000554084778
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,4,128,1,fp8,fp8,0,0.013022400438785553
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,16,1,128,1,float16,fp8,0,0.20108320713043212
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,8,128,1,float16,float16,0,0.012916800379753113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,8,128,1,float16,fp8,0,0.01334560066461563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,16,128,1,float16,float16,0,0.012489599734544754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,16,128,1,float16,fp8,0,0.012899200618267059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,16,128,1,fp8,fp8,0,0.012899200618267059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,1,128,1,float16,float16,0,0.012188799679279327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,1,128,1,float16,fp8,0,0.012824000418186187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,1,128,1,fp8,fp8,0,0.013400000333786011
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,2,128,1,float16,float16,0,0.012342400103807449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,16,2,128,1,float16,fp8,0,0.013859200477600097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,2,128,1,float16,fp8,0,0.012928000092506409
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,2,128,1,fp8,fp8,0,0.012775999307632447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,4,128,1,float16,float16,0,0.01255040019750595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,4,128,1,float16,fp8,0,0.012716799974441528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,16,1,128,1,float16,float16,0,0.10037120580673217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,8,128,1,float16,float16,0,0.012359999865293504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,8,128,1,float16,fp8,0,0.012905600666999816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,8,128,1,fp8,fp8,0,0.012801599502563477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,16,1,128,1,float16,float16,0,0.10087840557098389
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,16,1,128,1,float16,fp8,0,0.10882560014724732
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,16,128,1,float16,fp8,0,0.013203200697898865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,1,128,1,float16,float16,0,0.012449599802494049
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,16,1,128,1,fp8,fp8,0,0.10888160467147827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,16,2,128,1,float16,float16,0,0.1039423942565918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,16,2,128,1,float16,fp8,0,0.10851039886474609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,4,128,1,float16,fp8,0,0.013104000687599182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,16,4,128,1,float16,fp8,0,0.11206560134887696
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,16,4,128,1,fp8,fp8,0,0.11345280408859253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,16,8,128,1,fp8,fp8,0,0.013209599256515502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,16,8,128,1,float16,fp8,0,0.12534879446029662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,16,8,128,1,fp8,fp8,0,0.124835205078125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,16,128,1,float16,float16,0,0.11859359741210937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,16,128,1,float16,fp8,0,0.11378560066223145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,1,128,1,float16,float16,0,0.057918399572372437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,1,128,1,float16,fp8,0,0.0597104012966156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,1,128,1,fp8,fp8,0,0.059913599491119386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,16,4,128,1,fp8,fp8,0,0.012904000282287598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,2,128,1,float16,float16,0,0.05930079817771912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,2,128,1,float16,fp8,0,0.0604095995426178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,2,128,1,fp8,fp8,0,0.06006559729576111
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,4,128,1,float16,float16,0,0.064273601770401
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,4,128,1,float16,fp8,0,0.06167200207710266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,4,128,1,fp8,fp8,0,0.062159997224807736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,8,128,1,float16,float16,0,0.07161440253257752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,8,128,1,float16,fp8,0,0.06698079705238343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,8,128,1,fp8,fp8,0,0.06711040139198303
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,16,128,1,float16,float16,0,0.06613600254058838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,16,128,1,float16,fp8,0,0.06027200222015381
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,16,128,1,fp8,fp8,0,0.06037920117378235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,1,128,1,float16,float16,0,0.028865599632263185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,1,128,1,float16,fp8,0,0.031094399094581605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,1,128,1,fp8,fp8,0,0.030900800228118898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,2,128,1,float16,float16,0,0.030022400617599487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,2,128,1,float16,fp8,0,0.031027200818061828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,2,128,1,fp8,fp8,0,0.031147199869155883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,4,128,1,float16,float16,0,0.036504000425338745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,4,128,1,float16,fp8,0,0.03156639933586121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,4,128,1,fp8,fp8,0,0.030926400423049928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,8,128,1,float16,float16,0,0.04181439876556396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,8,128,1,float16,fp8,0,0.03471840023994446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,16,4,128,1,float16,float16,0,0.1123136043548584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,16,128,1,float16,float16,0,0.03664799928665161
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,16,128,1,float16,fp8,0,0.03030399978160858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,16,128,1,fp8,fp8,0,0.030257600545883178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,1,128,1,float16,float16,0,0.018799999356269838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,1,128,1,float16,fp8,0,0.019655999541282655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,1,128,1,fp8,fp8,0,0.019675199687480927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,2,128,1,float16,float16,0,0.018966400623321535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,2,128,1,float16,fp8,0,0.019809600710868836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,2,128,1,fp8,fp8,0,0.01976960003376007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,4,128,1,float16,float16,0,0.01943040043115616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,4,128,1,float16,fp8,0,0.019897599518299103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,4,128,1,fp8,fp8,0,0.01974239945411682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,8,128,1,float16,float16,0,0.020742399990558623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,8,128,1,float16,fp8,0,0.02008160054683685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,16,128,1,float16,float16,0,0.018643200397491455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,16,16,128,1,fp8,fp8,0,0.11313439607620239
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,16,128,1,float16,fp8,0,0.01870879977941513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,16,128,1,fp8,fp8,0,0.018779200315475465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,1,128,1,float16,float16,0,0.013451200723648072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,1,128,1,fp8,fp8,0,0.013716800510883332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,2,128,1,float16,float16,0,0.013484799861907959
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,2,128,1,fp8,fp8,0,0.014052799344062806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,4,128,1,float16,float16,0,0.013691200315952301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,4,128,1,float16,fp8,0,0.013964800536632538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,4,128,1,fp8,fp8,0,0.013982400298118591
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,8,128,1,float16,float16,0,0.014315199851989747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,8,128,1,float16,fp8,0,0.01435839980840683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,8,128,1,fp8,fp8,0,0.014345599710941315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,16,128,1,float16,float16,0,0.013575999438762665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,16,128,1,float16,fp8,0,0.013755199313163758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,16,128,1,fp8,fp8,0,0.013654400408267976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,1,128,1,float16,float16,0,0.012510399520397186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,1,128,1,float16,fp8,0,0.013457599282264709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,1,128,1,fp8,fp8,0,0.013176000118255616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,2,128,1,float16,float16,0,0.012755200266838074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,2,128,1,float16,fp8,0,0.013321599364280701
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,2,128,1,fp8,fp8,0,0.01337919980287552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,16,2,128,1,fp8,fp8,0,0.10988320112228393
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,4,128,1,float16,float16,0,0.012860800325870513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,16,8,128,1,fp8,fp8,0,0.034160000085830686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,4,128,1,fp8,fp8,0,0.013689599931240082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,8,128,1,float16,float16,0,0.013366399705410004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,8,128,1,fp8,fp8,0,0.013713599741458892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,16,8,128,1,float16,float16,0,0.12880480289459229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,16,128,1,float16,float16,0,0.01271200031042099
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,16,128,1,fp8,fp8,0,0.013064000010490417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,1,128,1,float16,float16,0,0.01239679977297783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,1,128,1,float16,fp8,0,0.013422399759292603
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,1,128,1,fp8,fp8,0,0.01319040060043335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,16,8,128,1,fp8,fp8,0,0.020236800611019134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,2,128,1,float16,float16,0,0.012427199631929398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,2,128,1,float16,fp8,0,0.01316159963607788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,1,128,1,float16,fp8,0,0.013760000467300415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,16,2,128,1,float16,fp8,0,0.013713599741458892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,4,128,1,float16,fp8,0,0.013040000200271606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,4,128,1,fp8,fp8,0,0.013040000200271606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,8,128,1,float16,float16,0,0.012644800543785095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,8,128,1,float16,fp8,0,0.013302400708198547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,8,128,1,fp8,fp8,0,0.013238400220870972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,16,128,1,float16,float16,0,0.012243200093507767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,16,128,1,float16,fp8,0,0.013009600341320038
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,16,128,1,fp8,fp8,0,0.013040000200271606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,1,128,1,float16,float16,0,0.012358400225639343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,16,1,128,1,float16,float16,0,0.02905600070953369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,1,128,1,float16,fp8,0,0.01310880035161972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,1,128,1,fp8,fp8,0,0.012881599366664886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,2,128,1,float16,fp8,0,0.01290079951286316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,2,128,1,fp8,fp8,0,0.012902399897575379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,4,128,1,float16,float16,0,0.012392000108957291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,4,128,1,float16,fp8,0,0.01292479932308197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,4,128,1,fp8,fp8,0,0.012780800461769104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,8,128,1,float16,float16,0,0.012252800166606903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,8,128,1,float16,fp8,0,0.012955200672149659
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,8,128,1,fp8,fp8,0,0.012803199887275695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,16,128,1,float16,float16,0,0.012160000205039979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,8,128,1,float16,fp8,0,0.013542400300502777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,16,128,1,float16,fp8,0,0.012772800028324127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,16,128,1,fp8,fp8,0,0.012604799866676331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,1,128,1,float16,float16,0,0.012240000069141388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,1,128,1,float16,fp8,0,0.012513600289821625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,16,128,1,float16,fp8,0,0.013305599987506866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,2,128,1,float16,float16,0,0.011902400106191636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,2,128,1,float16,fp8,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,2,128,1,fp8,fp8,0,0.012651200592517852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,4,128,1,float16,float16,0,0.011847999691963196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,4,128,1,float16,fp8,0,0.012643200159072877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,4,128,1,fp8,fp8,0,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,8,128,1,float16,float16,0,0.012041600048542022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,8,128,1,float16,fp8,0,0.012558400630950928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,2,128,1,fp8,fp8,0,0.013225600123405457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,16,1,128,1,float16,float16,0,0.05896000266075134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,16,4,128,1,float16,float16,0,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,16,1,128,1,float16,fp8,0,0.061337602138519284
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,16,1,128,1,fp8,fp8,0,0.06065120100975037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,16,2,128,1,float16,float16,0,0.060335999727249144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,16,2,128,1,float16,fp8,0,0.06153119802474975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,16,4,128,1,float16,float16,0,0.06452959775924683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,16,4,128,1,float16,fp8,0,0.06333119869232177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,16,4,128,1,fp8,fp8,0,0.063155198097229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,16,8,128,1,float16,float16,0,0.10306559801101685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,16,8,128,1,float16,fp8,0,0.10347520112991333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,16,8,128,1,fp8,fp8,0,0.1044160008430481
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,16,128,1,float16,fp8,0,0.09896000027656555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,16,2,128,1,float16,float16,0,0.0122079998254776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,1,128,1,float16,fp8,0,0.03213280141353607
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,1,128,1,fp8,fp8,0,0.03228000104427338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,2,128,1,float16,float16,0,0.03096800148487091
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,2,128,1,float16,fp8,0,0.03206880092620849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,2,128,1,fp8,fp8,0,0.0321615993976593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,16,4,128,1,float16,fp8,0,0.01377280056476593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,4,128,1,float16,fp8,0,0.03261120021343231
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,4,128,1,fp8,fp8,0,0.03224320113658905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,1,128,1,fp8,fp8,0,0.01271360069513321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,8,128,1,float16,float16,0,0.05699679851531982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,8,128,1,float16,fp8,0,0.053572797775268556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,8,128,1,fp8,fp8,0,0.05261759757995606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,16,128,1,float16,float16,0,0.05136479735374451
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,16,128,1,fp8,fp8,0,0.04925279915332794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,1,128,1,float16,float16,0,0.01865279972553253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,1,128,1,float16,fp8,0,0.02014400064945221
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,1,128,1,fp8,fp8,0,0.020131200551986694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,2,128,1,float16,float16,0,0.018926399946212768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,16,2,128,1,fp8,fp8,0,0.06164000034332275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,2,128,1,float16,fp8,0,0.02003680020570755
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,2,128,1,fp8,fp8,0,0.02005600035190582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,4,128,1,float16,float16,0,0.0192671999335289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,4,128,1,float16,fp8,0,0.020235200226306916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,4,128,1,fp8,fp8,0,0.020185600221157073
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,8,128,1,float16,float16,0,0.02776640057563782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,8,128,1,float16,fp8,0,0.02932479977607727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,8,128,1,fp8,fp8,0,0.029374399781227113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,16,128,1,float16,float16,0,0.026851201057434083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,16,128,1,float16,float16,0,0.09553920030593872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,16,128,1,float16,fp8,0,0.028352001309394838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,16,128,1,fp8,fp8,0,0.02847360074520111
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,1,128,1,float16,float16,0,0.013382400572299957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,1,128,1,fp8,fp8,0,0.01443839967250824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,2,128,1,float16,float16,0,0.013631999492645264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,2,128,1,float16,fp8,0,0.014646400511264802
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,1,128,1,float16,float16,0,0.030089598894119263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,4,128,1,float16,float16,0,0.014343999326229095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,4,128,1,float16,fp8,0,0.014312000572681427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,4,128,1,fp8,fp8,0,0.014611199498176575
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,8,128,1,float16,float16,0,0.017873600125312805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,8,128,1,float16,fp8,0,0.01910720020532608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,8,128,1,fp8,fp8,0,0.01880960017442703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,16,128,1,float16,float16,0,0.0173567995429039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,16,128,1,float16,fp8,0,0.018334400653839112
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,16,128,1,fp8,fp8,0,0.018115200102329254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,1,128,1,float16,float16,0,0.013158400356769562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,1,128,1,float16,fp8,0,0.013569599390029908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,1,128,1,fp8,fp8,0,0.013785600662231445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,2,128,1,float16,float16,0,0.012988799810409546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,2,128,1,float16,fp8,0,0.013700799643993377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,4,128,1,float16,float16,0,0.0366703987121582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,2,128,1,fp8,fp8,0,0.013889600336551667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,4,128,1,float16,float16,0,0.013219200074672699
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,4,128,1,float16,fp8,0,0.013817599415779114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,4,128,1,fp8,fp8,0,0.013724799454212188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,8,128,1,float16,float16,0,0.013275200128555298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,8,128,1,float16,fp8,0,0.013784000277519226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,16,8,128,1,fp8,fp8,0,0.013643200695514678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,16,128,1,float16,float16,0,0.013043199479579926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,16,128,1,float16,fp8,0,0.01340319961309433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,16,128,1,fp8,fp8,0,0.013491199910640716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,1,128,1,float16,float16,0,0.012580800056457519
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,1,128,1,float16,fp8,0,0.013380800187587739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,16,16,128,1,float16,fp8,0,0.04902240037918091
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,2,128,1,float16,float16,0,0.012561599910259246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,2,128,1,float16,fp8,0,0.01329440027475357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,2,128,1,fp8,fp8,0,0.013091200590133667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,4,128,1,float16,float16,0,0.012705600261688233
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,4,128,1,float16,fp8,0,0.01302880048751831
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,4,128,1,fp8,fp8,0,0.01316480040550232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,8,128,1,float16,float16,0,0.012574400007724761
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,8,128,1,float16,fp8,0,0.01297599971294403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,8,128,1,fp8,fp8,0,0.013212800025939941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,16,128,1,float16,float16,0,0.012545600533485413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,16,128,1,float16,fp8,0,0.013150399923324585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,16,128,1,fp8,fp8,0,0.013030399382114411
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,1,128,1,float16,float16,0,0.01202400028705597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,1,128,1,float16,fp8,0,0.012908799946308136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,1,128,1,fp8,fp8,0,0.012665599584579468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,2,128,1,float16,float16,0,0.012161599844694138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,2,128,1,float16,fp8,0,0.012715199589729309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,2,128,1,fp8,fp8,0,0.012915199995040894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,4,128,1,float16,float16,0,0.012052799761295318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,4,128,1,float16,fp8,0,0.012859199941158295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,4,128,1,fp8,fp8,0,0.012724800407886505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,8,128,1,float16,float16,0,0.011998400092124939
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,8,128,1,float16,fp8,0,0.01297439932823181
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,16,8,128,1,fp8,fp8,0,0.012694400548934937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,16,128,1,float16,float16,0,0.01234079971909523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,16,128,1,float16,fp8,0,0.01273919939994812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,1,128,1,float16,fp8,0,0.014212800562381745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,1,128,1,float16,float16,0,0.012030400335788727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,1,128,1,float16,fp8,0,0.012561599910259246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,1,128,1,fp8,fp8,0,0.01286720037460327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,16,2,128,1,fp8,fp8,0,0.014363199472427368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,2,128,1,float16,float16,0,0.011849600076675414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,2,128,1,fp8,fp8,0,0.01281919926404953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,4,128,1,float16,fp8,0,0.012848000228404998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,4,128,1,fp8,fp8,0,0.012838399410247803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,8,128,1,float16,float16,0,0.011959999799728394
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,8,128,1,float16,fp8,0,0.012608000636100769
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,8,128,1,fp8,fp8,0,0.012624000012874604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,16,128,1,float16,float16,0,0.011955200135707856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,16,128,1,float16,fp8,0,0.012435200065374375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,16,128,1,fp8,fp8,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,1,128,1,float16,float16,0,0.012043199688196182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,1,128,1,float16,fp8,0,0.012753599882125854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,1,128,1,fp8,fp8,0,0.012731200456619263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,2,128,1,float16,float16,0,0.012070400267839431
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,2,128,1,float16,fp8,0,0.012636800110340119
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,2,128,1,fp8,fp8,0,0.012625600397586822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,16,1,128,1,fp8,fp8,0,0.01321599930524826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,4,128,1,float16,float16,0,0.011849600076675414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,4,128,1,float16,fp8,0,0.012691199779510498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,4,128,1,fp8,fp8,0,0.012596799433231354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,8,128,1,float16,float16,0,0.011947199702262878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,8,128,1,fp8,fp8,0,0.012734399735927581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,16,1,128,1,float16,float16,0,0.03187359869480133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,16,1,128,1,float16,fp8,0,0.03434399962425232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,16,1,128,1,fp8,fp8,0,0.03412159979343414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,16,2,128,1,float16,float16,0,0.03328000009059906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,16,2,128,1,float16,fp8,0,0.03434560000896454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,16,2,128,1,fp8,fp8,0,0.03449119925498963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,16,4,128,1,float16,float16,0,0.05318719744682312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,16,4,128,1,float16,fp8,0,0.05220639705657959
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,16,16,128,1,fp8,fp8,0,0.09774240255355834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,2,128,1,float16,fp8,0,0.01281599998474121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,16,4,128,1,fp8,fp8,0,0.052127999067306516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,4,128,1,float16,float16,0,0.011795199662446975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,16,8,128,1,float16,float16,0,0.08879200220108033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,16,128,1,float16,float16,0,0.08296480178833007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,16,128,1,float16,fp8,0,0.08608480095863343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,16,128,1,fp8,fp8,0,0.0865440011024475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,1,128,1,float16,fp8,0,0.021104000508785248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,1,128,1,fp8,fp8,0,0.021423999965190888
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,2,128,1,float16,float16,0,0.019683200120925903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,2,128,1,float16,fp8,0,0.021427200734615327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,2,128,1,fp8,fp8,0,0.020916800200939178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,4,128,1,float16,float16,0,0.027651199698448183
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,4,128,1,float16,fp8,0,0.029951998591423036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,4,128,1,fp8,fp8,0,0.03033599853515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,8,128,1,float16,float16,0,0.04338879883289337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,8,128,1,float16,fp8,0,0.0483951985836029
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,8,128,1,fp8,fp8,0,0.0482015997171402
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,16,128,1,float16,float16,0,0.04166559875011444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,16,128,1,float16,fp8,0,0.04716480076313019
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,16,128,1,fp8,fp8,0,0.04726560115814209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,1,128,1,float16,float16,0,0.014582400023937226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,1,128,1,float16,fp8,0,0.014825600385665893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,1,128,1,fp8,fp8,0,0.015009599924087524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,2,128,1,float16,float16,0,0.014710399508476257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,16,8,128,1,fp8,fp8,0,0.012614400684833526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,2,128,1,float16,fp8,0,0.015372799336910247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,16,8,128,1,float16,fp8,0,0.01241919994354248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,2,128,1,fp8,fp8,0,0.015003199875354766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,4,128,1,float16,fp8,0,0.019204799830913544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,8,128,1,float16,float16,0,0.025923201441764833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,8,128,1,float16,fp8,0,0.028324800729751586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,8,128,1,fp8,fp8,0,0.028332799673080444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,16,128,1,float16,float16,0,0.02568320035934448
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,16,128,1,float16,fp8,0,0.02783679962158203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,16,128,1,fp8,fp8,0,0.027783998847007753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,1,128,1,float16,float16,0,0.013505600392818451
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,1,128,1,float16,fp8,0,0.014000000059604644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,16,16,128,1,fp8,fp8,0,0.013017599284648896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,1,128,1,fp8,fp8,0,0.013963200151920319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,2,128,1,float16,float16,0,0.01340479999780655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,2,128,1,fp8,fp8,0,0.0143327996134758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,4,128,1,float16,float16,0,0.013318400084972381
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,16,8,128,1,float16,fp8,0,0.0920415997505188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,4,128,1,float16,fp8,0,0.013867199420928955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,4,128,1,fp8,fp8,0,0.014129599928855896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,16,1,128,1,float16,float16,0,0.020033599436283113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,8,128,1,float16,fp8,0,0.018569600582122803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,8,128,1,fp8,fp8,0,0.018535999953746794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,16,128,1,float16,float16,0,0.01658879965543747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,16,128,1,float16,fp8,0,0.017788800597190856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,16,128,1,fp8,fp8,0,0.017664000391960144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,1,128,1,float16,float16,0,0.012656000256538392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,1,128,1,float16,fp8,0,0.013174399733543396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,1,128,1,fp8,fp8,0,0.013361600041389466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,2,128,1,float16,float16,0,0.012297599762678146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,2,128,1,float16,fp8,0,0.013344000279903411
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,2,128,1,fp8,fp8,0,0.013092799484729767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,4,128,1,float16,float16,0,0.012507200241088867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,4,128,1,fp8,fp8,0,0.013251200318336487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,8,128,1,float16,float16,0,0.01276639997959137
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,8,128,1,float16,fp8,0,0.013422399759292603
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,8,128,1,fp8,fp8,0,0.013283200562000275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,16,128,1,float16,float16,0,0.012611199915409089
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,16,128,1,float16,fp8,0,0.013316799700260163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,16,128,1,fp8,fp8,0,0.013324800133705138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,1,128,1,float16,float16,0,0.012408000230789185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,1,128,1,float16,fp8,0,0.012967999279499053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,1,128,1,fp8,fp8,0,0.012967999279499053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,2,128,1,float16,float16,0,0.01231520026922226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,2,128,1,float16,fp8,0,0.013201600313186646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,2,128,1,fp8,fp8,0,0.012972800433635712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,4,128,1,float16,float16,0,0.01231200024485588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,4,128,1,float16,fp8,0,0.01292639970779419
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,4,128,1,fp8,fp8,0,0.013060800731182098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,8,128,1,float16,float16,0,0.0124208003282547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,8,128,1,float16,fp8,0,0.01302880048751831
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,16,8,128,1,fp8,fp8,0,0.013020800054073333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,16,128,1,float16,float16,0,0.012302400171756744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,16,128,1,float16,fp8,0,0.013011200726032257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,16,128,1,fp8,fp8,0,0.012836800515651703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,1,128,1,float16,float16,0,0.012272000312805176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,1,128,1,float16,fp8,0,0.01265919953584671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,1,128,1,fp8,fp8,0,0.01279039978981018
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,2,128,1,float16,float16,0,0.012142399698495865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,2,128,1,float16,fp8,0,0.012785600125789642
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,2,128,1,fp8,fp8,0,0.01284160017967224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,4,128,1,float16,float16,0,0.011913599818944931
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,4,128,1,float16,fp8,0,0.012812800705432892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,4,128,1,fp8,fp8,0,0.01273760050535202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,8,128,1,float16,float16,0,0.012206400185823441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,8,128,1,float16,fp8,0,0.01279519945383072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,16,8,128,1,fp8,fp8,0,0.09212160110473633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,16,8,128,1,fp8,fp8,0,0.012729600071907043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,8,128,1,float16,float16,0,0.017392000555992125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,16,128,1,float16,fp8,0,0.012723200023174286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,1,128,1,float16,float16,0,0.011883199959993363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,1,128,1,float16,fp8,0,0.012892800569534301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,1,128,1,fp8,fp8,0,0.012615999579429627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,2,128,1,float16,float16,0,0.012057600170373916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,2,128,1,float16,fp8,0,0.012566399574279786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,2,128,1,fp8,fp8,0,0.012671999633312225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,4,128,1,float16,float16,0,0.01186719983816147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,4,128,1,float16,fp8,0,0.012479999661445617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,4,128,1,fp8,fp8,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,8,128,1,float16,float16,0,0.011793600022792816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,8,128,1,float16,fp8,0,0.012532800436019897
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,8,128,1,fp8,fp8,0,0.012561599910259246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,16,4,128,1,float16,fp8,0,0.013067199289798737
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,16,128,1,float16,fp8,0,0.0124719999730587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,16,128,1,fp8,fp8,0,0.012481600046157837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,1,128,1,float16,fp8,0,0.012646399438381195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,1,128,1,fp8,fp8,0,0.012625600397586822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,2,128,1,float16,float16,0,0.011790399998426437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,2,128,1,float16,fp8,0,0.012628799676895142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,2,128,1,fp8,fp8,0,0.012460800260305405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,4,128,1,float16,float16,0,0.018011200428009033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,16,4,128,1,fp8,fp8,0,0.019359999895095827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,4,128,1,fp8,fp8,0,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,8,128,1,float16,float16,0,0.011524800211191177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,8,128,1,float16,fp8,0,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,8,128,1,fp8,fp8,0,0.01242400035262108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,16,1,128,1,float16,float16,0,0.021196800470352172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,16,2,128,1,float16,fp8,0,0.01417119950056076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,16,1,128,1,float16,fp8,0,0.022444799542427063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,16,1,128,1,fp8,fp8,0,0.022438399493694305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,16,128,1,float16,float16,0,0.012281599640846252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,16,16,128,1,fp8,fp8,0,0.01265919953584671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,16,2,128,1,float16,fp8,0,0.03120799958705902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,16,2,128,1,fp8,fp8,0,0.03102239966392517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,16,4,128,1,float16,float16,0,0.04307360053062439
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,16,4,128,1,float16,fp8,0,0.04902719855308533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,16,4,128,1,fp8,fp8,0,0.04903360009193421
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,16,8,128,1,float16,float16,0,0.07113119959831238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,16,8,128,1,fp8,fp8,0,0.08447840213775634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,1,128,1,float16,float16,0,0.011777599900960922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,16,128,1,float16,fp8,0,0.08253920078277588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,16,128,1,fp8,fp8,0,0.08249920010566711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,1,128,1,float16,float16,0,0.01459999978542328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,4,128,1,float16,float16,0,0.011532799899578094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,4,128,1,float16,fp8,0,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,2,128,1,float16,float16,0,0.018409599363803864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,2,128,1,float16,fp8,0,0.019977599382400513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,2,128,1,fp8,fp8,0,0.02009280025959015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,4,128,1,float16,float16,0,0.026073598861694337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,4,128,1,float16,fp8,0,0.029126399755477907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,4,128,1,fp8,fp8,0,0.0290336012840271
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,8,128,1,float16,float16,0,0.04059039950370789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,8,128,1,float16,fp8,0,0.04652000069618225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,8,128,1,fp8,fp8,0,0.04645920097827912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,16,128,1,float16,float16,0,0.03918080031871796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,16,128,1,float16,fp8,0,0.045444801449775696
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,16,2,128,1,float16,float16,0,0.028118398785591126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,16,128,1,fp8,fp8,0,0.04545600116252899
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,1,128,1,float16,float16,0,0.0135343998670578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,1,128,1,float16,fp8,0,0.013945600390434265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,1,128,1,fp8,fp8,0,0.014236800372600555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,2,128,1,float16,float16,0,0.013260799646377563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,2,128,1,float16,fp8,0,0.014233599603176116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,4,128,1,float16,float16,0,0.017475199699401856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,16,8,128,1,float16,fp8,0,0.08479359745979309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,4,128,1,fp8,fp8,0,0.01892320066690445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,16,16,128,1,float16,float16,0,0.011884800344705581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,8,128,1,float16,float16,0,0.02491839975118637
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,8,128,1,float16,fp8,0,0.027492800354957582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,8,128,1,fp8,fp8,0,0.02762719988822937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,16,128,1,float16,float16,0,0.02459519952535629
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,16,128,1,float16,fp8,0,0.027110400795936584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,16,128,1,fp8,fp8,0,0.027220800518989563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,1,128,1,float16,float16,0,0.012593600153923034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,1,128,1,float16,fp8,0,0.01356000006198883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,2,128,1,float16,float16,0,0.01279360055923462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,2,128,1,float16,fp8,0,0.013558399677276612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,1,128,1,float16,fp8,0,0.015452800691127777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,2,128,1,fp8,fp8,0,0.013475200533866883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,1,128,1,fp8,fp8,0,0.015379199385643005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,4,128,1,float16,fp8,0,0.013489599525928497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,4,128,1,fp8,fp8,0,0.013697600364685059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,8,128,1,float16,float16,0,0.01658560037612915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,8,128,1,float16,fp8,0,0.018011200428009033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,16,128,1,float16,float16,0,0.0160288006067276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,16,128,1,float16,fp8,0,0.01719679981470108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,16,128,1,fp8,fp8,0,0.017164799571037292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,1,128,1,float16,float16,0,0.011649599671363831
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,1,128,1,float16,fp8,0,0.012385600060224534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,1,128,1,fp8,fp8,0,0.012531200051307678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,2,128,1,float16,float16,0,0.011950399726629257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,2,128,1,float16,fp8,0,0.012801599502563477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,2,128,1,fp8,fp8,0,0.012806400656700134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,4,128,1,float16,float16,0,0.012151999771595002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,4,128,1,float16,fp8,0,0.012801599502563477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,4,128,1,fp8,fp8,0,0.012639999389648438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,8,128,1,float16,float16,0,0.011975999921560287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,8,128,1,float16,fp8,0,0.012910400331020356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,16,8,128,1,fp8,fp8,0,0.013031999766826629
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,2,128,1,fp8,fp8,0,0.014110399782657624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,16,128,1,fp8,fp8,0,0.012936000525951386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,16,4,128,1,float16,fp8,0,0.018852800130844116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,1,128,1,float16,fp8,0,0.012403199821710587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,1,128,1,fp8,fp8,0,0.012392000108957291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,16,16,128,1,float16,float16,0,0.06958400011062622
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,2,128,1,float16,fp8,0,0.01276479959487915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,2,128,1,fp8,fp8,0,0.012591999769210816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,4,128,1,float16,float16,0,0.01204639971256256
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,1,128,1,fp8,fp8,0,0.01334560066461563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,4,128,1,float16,fp8,0,0.012814399600028992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,4,128,1,fp8,fp8,0,0.012916800379753113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,8,128,1,float16,float16,0,0.012043199688196182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,8,128,1,float16,fp8,0,0.013076800107955932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,4,128,1,float16,float16,0,0.012707200646400452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,16,128,1,float16,fp8,0,0.012775999307632447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,16,128,1,fp8,fp8,0,0.012873600423336028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,1,128,1,float16,float16,0,0.011563199758529662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,1,128,1,float16,fp8,0,0.012374400347471236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,1,128,1,fp8,fp8,0,0.012368000298738479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,2,128,1,float16,float16,0,0.011503999680280685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,2,128,1,float16,fp8,0,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,2,128,1,fp8,fp8,0,0.012190400063991547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,4,128,1,float16,float16,0,0.011593600362539291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,4,128,1,float16,fp8,0,0.012303999811410903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,4,128,1,fp8,fp8,0,0.012294399738311767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,8,128,1,float16,float16,0,0.011910399794578553
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,8,128,1,float16,fp8,0,0.012859199941158295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,16,128,1,float16,float16,0,0.012406399846076966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,16,128,1,float16,fp8,0,0.013129599392414093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,16,128,1,float16,float16,0,0.012054400146007537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,1,128,1,float16,float16,0,0.011689600348472596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,16,128,1,fp8,fp8,0,0.012806400656700134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,1,128,1,float16,float16,0,0.011244799941778183
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,2,128,1,float16,float16,0,0.012004800140857697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,1,128,1,float16,fp8,0,0.011878400295972823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,1,128,1,fp8,fp8,0,0.012176000326871873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,2,128,1,float16,fp8,0,0.012214399874210358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,2,128,1,fp8,fp8,0,0.012188799679279327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,4,128,1,float16,float16,0,0.011497599631547928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,4,128,1,float16,fp8,0,0.012164799869060517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,4,128,1,fp8,fp8,0,0.012292800098657608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,8,128,1,float16,float16,0,0.011798399686813354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,16,128,1,float16,float16,0,0.012251199781894683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,8,128,1,float16,fp8,0,0.012601600587368011
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,8,128,1,fp8,fp8,0,0.012548799812793731
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,16,128,1,float16,float16,0,0.01199359968304634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,16,8,128,1,fp8,fp8,0,0.01788959950208664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,1,128,1,float16,float16,0,0.01138719990849495
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,1,128,1,float16,fp8,0,0.012104000151157378
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,1,128,1,fp8,fp8,0,0.012308800220489502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,2,128,1,float16,float16,0,0.01154559999704361
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,2,128,1,float16,fp8,0,0.01228799968957901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,2,128,1,fp8,fp8,0,0.012305600196123123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,4,128,1,float16,float16,0,0.011575999855995178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,4,128,1,float16,fp8,0,0.01218400001525879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,4,128,1,fp8,fp8,0,0.012124799937009812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,8,128,1,float16,float16,0,0.01138240024447441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,8,128,1,float16,fp8,0,0.012241599708795547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,8,128,1,fp8,fp8,0,0.012191999703645706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,16,8,128,1,fp8,fp8,0,0.012488000094890594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,16,128,1,float16,fp8,0,0.012515200674533844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,16,2,128,1,float16,float16,0,0.01130400002002716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,16,8,128,1,fp8,fp8,0,0.012812800705432892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,16,128,1,float16,fp8,0,0.012454400211572647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,16,16,128,1,fp8,fp8,0,0.012604799866676331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,12,1,128,1,float16,fp8,0,9.633612823486327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,12,1,128,1,fp8,fp8,0,9.474716949462891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,12,1,128,1,float16,float16,0,11.423426818847656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,12,2,128,1,float16,fp8,0,9.565987396240235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,12,2,128,1,fp8,fp8,0,9.59359359741211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,12,2,128,1,float16,float16,0,11.722003173828124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,12,4,128,1,float16,float16,0,11.73913116455078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,12,4,128,1,float16,fp8,0,9.697994995117188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,12,12,128,1,float16,float16,0,5.672177505493164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,12,1,128,1,float16,float16,0,5.596335983276367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,12,1,128,1,float16,fp8,0,4.9077392578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,12,1,128,1,fp8,fp8,0,4.880084609985351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,12,4,128,1,fp8,fp8,0,9.790025329589843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,12,2,128,1,float16,fp8,0,4.862265777587891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,12,2,128,1,float16,float16,0,5.670897674560547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,12,2,128,1,fp8,fp8,0,4.888659286499023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,12,4,128,1,float16,float16,0,5.837432098388672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,12,12,128,1,float16,float16,0,2.8309791564941404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,12,4,128,1,float16,fp8,0,4.872689437866211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,12,4,128,1,fp8,fp8,0,4.858291244506836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,12,1,128,1,float16,float16,0,2.8348432540893556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,12,1,128,1,float16,fp8,0,2.476068878173828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,12,1,128,1,fp8,fp8,0,2.370787239074707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,12,2,128,1,float16,float16,0,2.6361839294433596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,12,2,128,1,fp8,fp8,0,2.3919279098510744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,12,2,128,1,float16,fp8,0,2.6489519119262694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,12,4,128,1,float16,float16,0,2.710809516906738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,12,4,128,1,float16,fp8,0,2.633036804199219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,12,12,128,1,float16,float16,0,1.3328432083129882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,12,4,128,1,fp8,fp8,0,2.379855918884277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,12,1,128,1,float16,float16,0,1.3210176467895507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,12,1,128,1,float16,fp8,0,1.2091407775878906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,12,1,128,1,fp8,fp8,0,1.4036623954772949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,12,2,128,1,float16,float16,0,1.3539088249206543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,12,2,128,1,float16,fp8,0,1.3078736305236816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,12,2,128,1,fp8,fp8,0,1.482307243347168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,12,4,128,1,float16,float16,0,1.3687423706054687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,12,4,128,1,float16,fp8,0,1.2917200088500977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,12,4,128,1,fp8,fp8,0,1.4183360099792481
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,12,1,128,1,float16,fp8,0,5.55843505859375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,12,1,128,1,float16,float16,0,6.748857879638672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,12,1,128,1,fp8,fp8,0,5.603300857543945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,12,2,128,1,float16,float16,0,6.4696044921875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,12,2,128,1,float16,fp8,0,5.563857650756836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,12,2,128,1,fp8,fp8,0,5.528857421875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,12,4,128,1,float16,float16,0,6.866985321044922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,12,4,128,1,float16,fp8,0,5.50283203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,12,12,128,1,float16,float16,0,3.3638751983642576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,12,4,128,1,fp8,fp8,0,5.642793655395508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,12,12,128,1,float16,fp8,0,1.222011184692383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,12,12,128,1,float16,fp8,0,2.418840026855469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,12,12,128,1,fp8,fp8,0,2.4247711181640623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,12,12,128,1,float16,fp8,0,4.852238464355469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,12,1,128,1,float16,fp8,0,2.7101568222045898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,12,12,128,1,float16,fp8,0,2.7663408279418946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,12,1,128,1,float16,float16,0,3.1751855850219726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,12,12,128,1,fp8,fp8,0,4.893398284912109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,12,1,128,1,fp8,fp8,0,2.6894704818725588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,12,12,128,1,fp8,fp8,0,1.219167995452881
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,12,2,128,1,float16,float16,0,3.1792959213256835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,12,12,128,1,fp8,fp8,0,2.783608055114746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,12,2,128,1,float16,fp8,0,2.7164943695068358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,12,2,128,1,fp8,fp8,0,2.712166404724121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,12,12,128,1,float16,float16,0,1.6013727188110352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,12,12,128,1,float16,fp8,0,1.5464991569519042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,12,4,128,1,float16,float16,0,3.325651168823242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,12,4,128,1,float16,fp8,0,2.733603286743164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,12,4,128,1,fp8,fp8,0,2.7410831451416016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,12,12,128,1,fp8,fp8,0,1.4156175613403321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,12,1,128,1,float16,float16,0,1.4223664283752442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,12,1,128,1,float16,fp8,0,1.4402671813964845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,12,1,128,1,fp8,fp8,0,1.3836336135864258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,12,2,128,1,float16,float16,0,1.6010799407958984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,12,2,128,1,float16,fp8,0,1.3544912338256836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,12,2,128,1,fp8,fp8,0,1.3622943878173828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,12,4,128,1,float16,float16,0,1.5470640182495117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,12,4,128,1,float16,fp8,0,1.377729606628418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,12,4,128,1,fp8,fp8,0,1.3770992279052734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,12,12,128,1,float16,fp8,0,0.8121024131774902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,12,1,128,1,float16,float16,0,0.7209519863128662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,12,1,128,1,float16,fp8,0,0.7980175971984863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,12,1,128,1,fp8,fp8,0,0.6894336223602295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,12,2,128,1,float16,float16,0,0.7410287857055664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,12,2,128,1,float16,fp8,0,0.7345088005065918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,12,2,128,1,fp8,fp8,0,0.7562511920928955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,12,4,128,1,float16,float16,0,0.7885744094848632
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,12,4,128,1,float16,fp8,0,0.6979616165161133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,12,4,128,1,fp8,fp8,0,0.7099040031433106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,12,12,128,1,float16,float16,0,0.8154128074645997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,12,12,128,1,fp8,fp8,0,0.7333295822143555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,12,1,128,1,float16,fp8,0,3.813276672363281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,12,1,128,1,fp8,fp8,0,3.897020721435547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,12,2,128,1,float16,fp8,0,3.809280014038086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,12,2,128,1,float16,float16,0,4.47002067565918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,12,2,128,1,fp8,fp8,0,3.7836238861083986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,12,12,128,1,float16,fp8,0,2.0039295196533202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,12,12,128,1,float16,float16,0,2.213175964355469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,12,4,128,1,float16,fp8,0,3.8599006652832033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,12,4,128,1,float16,float16,0,4.41595344543457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,12,4,128,1,fp8,fp8,0,3.87036018371582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,12,1,128,1,float16,float16,0,4.578728103637696
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,12,1,128,1,float16,float16,0,2.1392335891723633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,12,1,128,1,float16,fp8,0,1.8862384796142577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,12,1,128,1,fp8,fp8,0,2.0764080047607423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,12,2,128,1,float16,float16,0,2.122243118286133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,12,2,128,1,float16,fp8,0,2.0702991485595703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,12,2,128,1,fp8,fp8,0,1.9149360656738281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,12,12,128,1,float16,float16,0,1.1408991813659668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,12,12,128,1,float16,fp8,0,1.1356592178344727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,12,4,128,1,float16,float16,0,2.187158393859863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,12,12,128,1,fp8,fp8,0,1.0361248016357423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,12,4,128,1,float16,fp8,0,2.1000911712646486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,12,4,128,1,fp8,fp8,0,1.935256004333496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,12,12,128,1,fp8,fp8,0,2.116032028198242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,12,1,128,1,float16,float16,0,1.0119791984558106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,12,1,128,1,float16,fp8,0,1.0126527786254882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,12,2,128,1,float16,fp8,0,0.9577199935913085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,12,2,128,1,fp8,fp8,0,0.976195240020752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,12,2,128,1,float16,float16,0,1.185097599029541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,12,4,128,1,float16,float16,0,1.05938720703125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,12,4,128,1,float16,fp8,0,0.9608880043029785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,12,12,128,1,float16,float16,0,0.6027376174926757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,12,4,128,1,fp8,fp8,0,0.9709312438964843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,12,12,128,1,float16,fp8,0,0.5241231918334961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,12,12,128,1,fp8,fp8,0,0.5325312137603759
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,12,1,128,1,float16,fp8,0,0.518552017211914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,12,1,128,1,fp8,fp8,0,0.4799647808074951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,12,2,128,1,float16,float16,0,0.5230016231536865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,12,2,128,1,float16,fp8,0,0.5360176086425781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,12,2,128,1,fp8,fp8,0,0.4818336009979248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,12,4,128,1,float16,float16,0,0.5244336128234863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,12,4,128,1,float16,fp8,0,0.5128367900848388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,12,4,128,1,fp8,fp8,0,0.4857615947723389
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,12,1,128,1,fp8,fp8,0,0.9966079711914062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,12,1,128,1,float16,float16,0,0.5968751907348633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,12,1,128,1,fp8,fp8,0,4.9803009033203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,12,2,128,1,fp8,fp8,0,4.946656036376953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,12,2,128,1,float16,fp8,0,5.043793487548828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,12,1,128,1,float16,float16,0,6.03145751953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,12,2,128,1,float16,float16,0,5.812176132202149
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,12,4,128,1,float16,float16,0,5.887558364868164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,12,12,128,1,float16,fp8,0,2.593497657775879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,12,12,128,1,float16,float16,0,2.973478317260742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,12,1,128,1,float16,fp8,0,4.966417694091797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,12,12,128,1,fp8,fp8,0,2.711884880065918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,12,1,128,1,float16,float16,0,2.9012992858886717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,12,4,128,1,fp8,fp8,0,5.035276794433594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,12,1,128,1,fp8,fp8,0,2.5094720840454103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,12,1,128,1,float16,fp8,0,2.6149391174316405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,12,2,128,1,float16,fp8,0,2.481372833251953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,12,2,128,1,fp8,fp8,0,2.5361663818359377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,12,2,128,1,float16,float16,0,2.894548797607422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,12,4,128,1,float16,float16,0,2.9384159088134765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,12,4,128,1,float16,fp8,0,5.013724899291992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,12,12,128,1,float16,float16,0,1.4486528396606446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,12,4,128,1,float16,fp8,0,2.51759033203125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,12,12,128,1,float16,fp8,0,1.316481590270996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,12,12,128,1,fp8,fp8,0,1.4351360321044921
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,12,1,128,1,float16,float16,0,1.4442543983459473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,12,4,128,1,fp8,fp8,0,2.5644336700439454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,12,1,128,1,float16,fp8,0,1.4345343589782715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,12,1,128,1,fp8,fp8,0,1.239030361175537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,12,2,128,1,float16,float16,0,1.4227439880371093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,12,2,128,1,float16,fp8,0,1.2560352325439452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,12,2,128,1,fp8,fp8,0,1.2458383560180664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,12,12,128,1,float16,float16,0,0.7159632205963135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,12,4,128,1,float16,fp8,0,1.3854479789733887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,12,12,128,1,float16,fp8,0,0.7214000225067139
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,12,4,128,1,fp8,fp8,0,1.3209152221679688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,12,1,128,1,float16,float16,0,0.6592720031738282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,12,1,128,1,float16,fp8,0,0.6372416019439697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,12,1,128,1,fp8,fp8,0,0.6448319911956787
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,12,2,128,1,float16,float16,0,0.677675199508667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,12,4,128,1,float16,float16,0,0.6780928134918213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,12,4,128,1,float16,fp8,0,0.6419871807098388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,12,4,128,1,fp8,fp8,0,0.689247989654541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,12,12,128,1,float16,fp8,0,0.3392047882080078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,12,12,128,1,fp8,fp8,0,0.349566388130188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,12,1,128,1,float16,float16,0,0.33280320167541505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,12,1,128,1,float16,fp8,0,0.32912321090698243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,12,1,128,1,fp8,fp8,0,0.31517601013183594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,12,2,128,1,float16,float16,0,0.3422271966934204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,12,2,128,1,float16,fp8,0,0.3165872097015381
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,12,2,128,1,fp8,fp8,0,0.31844000816345214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,12,4,128,1,float16,float16,0,0.33946239948272705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,12,4,128,1,float16,fp8,0,0.3216415882110596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,12,4,128,1,fp8,fp8,0,0.3202575922012329
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,12,2,128,1,float16,fp8,0,0.6531055927276611
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,12,12,128,1,float16,float16,0,0.37429759502410886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,12,1,128,1,float16,fp8,0,2.849518394470215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,12,1,128,1,float16,float16,0,3.284401702880859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,12,12,128,1,fp8,fp8,0,0.6594736099243164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,12,2,128,1,fp8,fp8,0,0.627732801437378
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,12,2,128,1,float16,float16,0,3.239886474609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,12,2,128,1,float16,fp8,0,2.863102340698242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,12,2,128,1,fp8,fp8,0,2.8725296020507813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,12,4,128,1,float16,fp8,0,2.897332763671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,12,12,128,1,float16,float16,0,1.8253807067871093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,12,4,128,1,float16,float16,0,3.449969482421875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,12,4,128,1,float16,float16,0,1.4940079689025878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,12,1,128,1,fp8,fp8,0,2.837414360046387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,12,4,128,1,fp8,fp8,0,2.8912208557128904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,12,12,128,1,float16,fp8,0,1.5095696449279785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,12,12,128,1,fp8,fp8,0,1.5183327674865723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,12,1,128,1,float16,float16,0,1.5208640098571777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,12,1,128,1,float16,fp8,0,1.5355152130126952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,12,1,128,1,fp8,fp8,0,1.5393919944763184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,12,2,128,1,float16,float16,0,1.5348624229431151
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,12,2,128,1,float16,fp8,0,1.4296591758728028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,12,2,128,1,fp8,fp8,0,1.4443856239318849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,12,4,128,1,float16,float16,0,1.6163471221923829
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,12,12,128,1,float16,fp8,0,0.7843247890472412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,12,12,128,1,fp8,fp8,0,0.8287216186523437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,12,1,128,1,float16,float16,0,0.7667471885681152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,12,1,128,1,float16,fp8,0,0.7247519969940186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,12,4,128,1,fp8,fp8,0,1.57916316986084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,12,1,128,1,fp8,fp8,0,0.8333104133605957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,12,2,128,1,float16,float16,0,0.770305585861206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,12,2,128,1,float16,fp8,0,0.7945280075073242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,12,2,128,1,fp8,fp8,0,0.7566864013671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,12,4,128,1,float16,float16,0,0.7804880142211914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,12,4,128,1,float16,fp8,0,0.7640831947326661
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,12,12,128,1,float16,float16,0,0.47370400428771975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,12,12,128,1,float16,fp8,0,0.427888011932373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,12,4,128,1,fp8,fp8,0,0.7715616226196289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,12,1,128,1,float16,float16,0,0.41905598640441893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,12,1,128,1,float16,fp8,0,0.37236800193786623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,12,1,128,1,fp8,fp8,0,0.38900320529937743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,12,2,128,1,float16,float16,0,0.3988800048828125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,12,2,128,1,float16,fp8,0,0.3758831977844238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,12,2,128,1,fp8,fp8,0,0.40411200523376467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,12,4,128,1,float16,fp8,0,0.38972320556640627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,12,4,128,1,fp8,fp8,0,0.37452800273895265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,12,12,128,1,fp8,fp8,0,0.2091536045074463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,12,1,128,1,float16,float16,0,0.20200319290161134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,12,1,128,1,float16,fp8,0,0.1934847950935364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,12,2,128,1,float16,float16,0,0.20434880256652832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,12,4,128,1,float16,fp8,0,1.4502191543579102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,12,2,128,1,float16,fp8,0,0.19065920114517212
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,12,2,128,1,fp8,fp8,0,0.19258559942245485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,12,4,128,1,float16,float16,0,0.20827999114990234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,12,4,128,1,float16,fp8,0,0.19304319620132446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,12,4,128,1,fp8,fp8,0,0.19116480350494386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,12,12,128,1,fp8,fp8,0,0.4072591781616211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,12,4,128,1,float16,float16,0,0.3994944095611572
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,12,12,128,1,float16,float16,0,0.24384000301361083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,12,12,128,1,float16,fp8,0,0.2226560115814209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,12,1,128,1,fp8,fp8,0,0.19044959545135498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,12,12,128,1,float16,float16,0,0.8604207992553711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,12,1,128,1,float16,float16,0,2.938515281677246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,12,1,128,1,fp8,fp8,0,2.6744768142700197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,12,2,128,1,float16,float16,0,2.9485328674316404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,12,2,128,1,float16,fp8,0,2.7017887115478514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,12,2,128,1,fp8,fp8,0,2.7014768600463865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,12,12,128,1,float16,float16,0,1.6211727142333985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,12,4,128,1,float16,float16,0,2.948203277587891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,12,4,128,1,float16,fp8,0,2.733006477355957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,12,4,128,1,fp8,fp8,0,2.85164794921875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,12,1,128,1,float16,float16,0,1.4193807601928712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,12,12,128,1,float16,fp8,0,1.5836463928222657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,12,1,128,1,float16,fp8,0,2.664219284057617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,12,1,128,1,float16,fp8,0,1.5043919563293457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,12,1,128,1,fp8,fp8,0,1.482102394104004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,12,2,128,1,float16,float16,0,1.4407872200012206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,12,2,128,1,float16,fp8,0,1.3676063537597656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,12,2,128,1,fp8,fp8,0,1.4462800025939941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,12,4,128,1,float16,float16,0,1.483084774017334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,12,4,128,1,float16,fp8,0,1.376968002319336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,12,12,128,1,float16,fp8,0,0.7392384052276612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,12,4,128,1,fp8,fp8,0,1.3972767829895019
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,12,1,128,1,float16,float16,0,0.730463981628418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,12,1,128,1,float16,fp8,0,0.6824719905853271
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,12,1,128,1,fp8,fp8,0,0.6810416221618653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,12,2,128,1,float16,float16,0,0.7232048034667968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,12,12,128,1,fp8,fp8,0,1.4541119575500487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,12,4,128,1,float16,float16,0,0.7309152126312256
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,12,4,128,1,float16,fp8,0,0.709659194946289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,12,4,128,1,fp8,fp8,0,0.6951488018035888
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,12,12,128,1,float16,fp8,0,0.3786384105682373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,12,12,128,1,fp8,fp8,0,0.37666079998016355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,12,1,128,1,float16,float16,0,0.3622879981994629
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,12,1,128,1,float16,fp8,0,0.34716799259185793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,12,1,128,1,fp8,fp8,0,0.3453711986541748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,12,12,128,1,float16,float16,0,0.8685744285583497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,12,2,128,1,float16,float16,0,0.3704848051071167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,12,2,128,1,float16,fp8,0,0.3458703994750977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,12,12,128,1,fp8,fp8,0,0.7524896144866944
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,12,2,128,1,fp8,fp8,0,0.3521487951278687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,12,4,128,1,float16,float16,0,0.3779072046279907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,12,12,128,1,float16,float16,0,0.21156480312347412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,12,4,128,1,fp8,fp8,0,0.3544719934463501
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,12,12,128,1,float16,fp8,0,0.1960271954536438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,12,12,128,1,fp8,fp8,0,0.19481760263442993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,12,2,128,1,float16,fp8,0,0.7150383949279785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,12,1,128,1,float16,fp8,0,0.1814560055732727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,12,2,128,1,float16,float16,0,0.188264000415802
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,12,2,128,1,fp8,fp8,0,0.6865312099456787
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,12,2,128,1,float16,fp8,0,0.18278720378875732
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,12,2,128,1,fp8,fp8,0,0.17900960445404052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,12,4,128,1,float16,float16,0,0.1931424021720886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,12,4,128,1,float16,fp8,0,0.1822559952735901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,12,4,128,1,fp8,fp8,0,0.18341120481491088
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,12,12,128,1,float16,fp8,0,0.1193120002746582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,12,12,128,1,fp8,fp8,0,0.11930719614028931
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,12,1,128,1,float16,float16,0,0.10311039686203002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,12,1,128,1,float16,fp8,0,0.09717599749565124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,12,1,128,1,fp8,fp8,0,0.09714879989624023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,12,2,128,1,float16,float16,0,0.1030351996421814
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,12,2,128,1,float16,fp8,0,0.09786559939384461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,12,2,128,1,fp8,fp8,0,0.0973471999168396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,12,4,128,1,float16,float16,0,0.10686240196228028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,12,4,128,1,float16,fp8,0,0.09836320281028747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,12,4,128,1,fp8,fp8,0,0.09800159931182861
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,12,4,128,1,float16,fp8,0,0.3499039888381958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,12,1,128,1,float16,fp8,0,1.5944463729858398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,12,1,128,1,fp8,fp8,0,1.604529571533203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,12,1,128,1,float16,float16,0,0.1864143967628479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,12,1,128,1,fp8,fp8,0,0.1781440019607544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,12,12,128,1,float16,float16,0,0.13179839849472047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,12,12,128,1,float16,float16,0,0.40610079765319823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,12,2,128,1,float16,fp8,0,1.6016223907470704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,12,2,128,1,fp8,fp8,0,1.6131376266479491
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,12,4,128,1,float16,float16,0,1.7214815139770507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,12,4,128,1,float16,fp8,0,1.6373760223388671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,12,12,128,1,float16,float16,0,0.9762928009033203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,12,12,128,1,float16,fp8,0,0.8955360412597656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,12,12,128,1,fp8,fp8,0,0.9018256187438964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,12,4,128,1,fp8,fp8,0,1.6394655227661132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,12,1,128,1,float16,float16,0,0.8929696083068848
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,12,1,128,1,float16,fp8,0,0.8055744171142578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,12,1,128,1,fp8,fp8,0,0.7993504047393799
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,12,2,128,1,float16,float16,0,0.8425200462341309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,12,2,128,1,float16,fp8,0,0.8062416076660156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,12,2,128,1,fp8,fp8,0,0.812497615814209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,12,4,128,1,float16,float16,0,0.9297871589660645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,12,4,128,1,float16,fp8,0,0.8260656356811523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,12,12,128,1,float16,float16,0,0.5008207798004151
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,12,12,128,1,fp8,fp8,0,0.46541438102722166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,12,4,128,1,fp8,fp8,0,0.8175087928771972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,12,1,128,1,float16,fp8,0,0.40987200736999513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,12,1,128,1,fp8,fp8,0,0.40522079467773436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,12,2,128,1,float16,float16,0,0.42846240997314455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,12,2,128,1,float16,fp8,0,0.4078976154327393
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,12,2,128,1,fp8,fp8,0,0.412721586227417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,12,4,128,1,float16,float16,0,0.43952322006225586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,12,4,128,1,fp8,fp8,0,0.41552319526672366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,12,12,128,1,float16,float16,0,0.26096320152282715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,12,12,128,1,float16,fp8,0,0.24398720264434814
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,12,12,128,1,fp8,fp8,0,0.2439296007156372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,12,1,128,1,float16,float16,0,0.21861441135406495
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,12,1,128,1,float16,fp8,0,0.20905120372772218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,12,1,128,1,fp8,fp8,0,0.2085200071334839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,12,2,128,1,float16,float16,0,0.2212912082672119
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,12,2,128,1,float16,fp8,0,0.21012320518493652
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,12,2,128,1,fp8,fp8,0,0.20977120399475097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,12,4,128,1,float16,float16,0,0.22787039279937743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,12,4,128,1,float16,fp8,0,0.21415839195251465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,12,4,128,1,fp8,fp8,0,0.21445760726928711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,12,12,128,1,float16,float16,0,0.1400928020477295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,12,12,128,1,float16,fp8,0,0.13550879955291747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,12,12,128,1,fp8,fp8,0,0.1358464002609253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,12,1,128,1,float16,float16,0,0.11457600593566894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,12,1,128,1,float16,fp8,0,0.10908000469207764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,12,1,128,1,fp8,fp8,0,0.10908960103988648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,12,2,128,1,float16,float16,0,0.11743359565734864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,12,2,128,1,fp8,fp8,0,0.10994080305099488
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,12,12,128,1,float16,fp8,0,0.462659215927124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,12,4,128,1,float16,float16,0,0.12154079675674438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,12,4,128,1,float16,fp8,0,0.11325119733810425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,12,4,128,1,fp8,fp8,0,0.11231679916381836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,12,1,128,1,float16,float16,0,0.45774078369140625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,12,12,128,1,float16,float16,0,0.08891040086746216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,12,12,128,1,float16,fp8,0,0.07928159832954407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,12,12,128,1,fp8,fp8,0,0.0787872016429901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,12,1,128,1,float16,float16,0,0.06403679847717285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,12,1,128,1,float16,fp8,0,0.06142560243606567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,12,1,128,1,fp8,fp8,0,0.06165279746055603
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,12,2,128,1,float16,float16,0,0.06445440053939819
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,12,2,128,1,float16,fp8,0,0.06148800253868103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,12,2,128,1,fp8,fp8,0,0.06209440231323242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,12,4,128,1,float16,fp8,0,0.06274719834327698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,12,4,128,1,float16,fp8,0,0.42784957885742186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,12,1,128,1,float16,float16,0,1.6387727737426758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,12,2,128,1,float16,fp8,0,0.10959359407424926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,12,2,128,1,float16,float16,0,1.697275161743164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,12,4,128,1,float16,float16,0,0.06555039882659912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,12,1,128,1,fp8,fp8,0,1.7184480667114257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,12,4,128,1,fp8,fp8,0,0.06331999897956848
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,12,2,128,1,float16,float16,0,1.6277008056640625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,12,2,128,1,float16,fp8,0,1.587551975250244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,12,2,128,1,fp8,fp8,0,1.5648223876953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,12,1,128,1,float16,fp8,0,1.571671962738037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,12,4,128,1,float16,float16,0,1.7490720748901367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,12,4,128,1,fp8,fp8,0,1.6833951950073243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,12,12,128,1,float16,float16,0,0.99957275390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,12,1,128,1,float16,float16,0,0.8156944274902344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,12,12,128,1,fp8,fp8,0,0.9038335800170898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,12,1,128,1,float16,fp8,0,0.846406364440918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,12,1,128,1,fp8,fp8,0,0.8005264282226563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,12,2,128,1,float16,float16,0,0.8639311790466309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,12,2,128,1,float16,fp8,0,0.7999392032623291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,12,4,128,1,float16,float16,0,0.8808064460754395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,12,4,128,1,float16,fp8,0,0.8195903778076172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,12,12,128,1,float16,float16,0,0.5137936115264893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,12,4,128,1,fp8,fp8,0,0.8603343963623047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,12,12,128,1,float16,fp8,0,0.47890558242797854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,12,12,128,1,fp8,fp8,0,0.47136478424072265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,12,1,128,1,float16,float16,0,0.4105391979217529
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,12,1,128,1,float16,fp8,0,0.42328319549560545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,12,1,128,1,fp8,fp8,0,0.40577921867370603
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,12,2,128,1,float16,float16,0,0.42119359970092773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,12,2,128,1,float16,fp8,0,0.40366239547729493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,12,4,128,1,float16,fp8,0,1.6275152206420898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,12,4,128,1,float16,float16,0,0.4338128089904785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,12,4,128,1,float16,fp8,0,0.41782560348510744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,12,4,128,1,fp8,fp8,0,0.413259220123291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,12,12,128,1,float16,fp8,0,0.24996960163116455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,12,1,128,1,float16,float16,0,0.21064000129699706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,12,1,128,1,float16,fp8,0,0.20913441181182862
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,12,12,128,1,float16,fp8,0,0.9163295745849609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,12,1,128,1,fp8,fp8,0,0.2061311960220337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,12,2,128,1,float16,float16,0,0.21641919612884522
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,12,2,128,1,float16,fp8,0,0.2060960054397583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,12,2,128,1,fp8,fp8,0,0.20759999752044678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,12,4,128,1,float16,float16,0,0.22306718826293945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,12,4,128,1,float16,fp8,0,0.21207680702209472
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,12,4,128,1,fp8,fp8,0,0.2132767915725708
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,12,12,128,1,float16,fp8,0,0.13568960428237914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,12,12,128,1,fp8,fp8,0,0.13708000183105468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,12,1,128,1,float16,fp8,0,0.10751359462738037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,12,1,128,1,fp8,fp8,0,0.10703999996185302
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,12,2,128,1,float16,float16,0,0.11334400177001953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,12,2,128,1,float16,fp8,0,0.10868480205535888
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,12,2,128,1,fp8,fp8,0,0.10868480205535888
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,12,4,128,1,float16,float16,0,0.11773120164871216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,12,2,128,1,fp8,fp8,0,0.42342238426208495
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,12,4,128,1,float16,fp8,0,0.11225759983062744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,12,4,128,1,fp8,fp8,0,0.11340960264205932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,12,12,128,1,float16,float16,0,0.08335520029067993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,12,12,128,1,float16,float16,0,0.27009921073913573
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,12,12,128,1,float16,fp8,0,0.07688639760017395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,12,1,128,1,float16,float16,0,0.06001600027084351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,12,12,128,1,fp8,fp8,0,0.24919838905334474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,12,1,128,1,fp8,fp8,0,0.059217602014541626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,12,2,128,1,float16,float16,0,0.06089760065078735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,12,2,128,1,float16,fp8,0,0.05945919752120972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,12,2,128,1,fp8,fp8,0,0.060043197870254514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,12,4,128,1,float16,float16,0,0.06298080086708069
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,12,4,128,1,float16,fp8,0,0.0602895975112915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,12,4,128,1,fp8,fp8,0,0.059708797931671144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,12,12,128,1,float16,float16,0,0.04440479874610901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,12,12,128,1,float16,fp8,0,0.04100480079650879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,12,12,128,1,fp8,fp8,0,0.041371199488639834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,12,1,128,1,float16,float16,0,0.0368943989276886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,12,1,128,1,float16,fp8,0,0.03675679862499237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,12,2,128,1,fp8,fp8,0,0.7875120162963867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,12,1,128,1,fp8,fp8,0,0.03631199896335602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,12,2,128,1,float16,float16,0,0.037215998768806456
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,12,2,128,1,fp8,fp8,0,0.035927999019622806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,12,1,128,1,float16,float16,0,0.1102944016456604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,12,4,128,1,float16,fp8,0,0.03645600080490112
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,12,4,128,1,fp8,fp8,0,0.036627200245857236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,12,1,128,1,float16,float16,0,0.9768048286437988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,12,1,128,1,float16,float16,0,1.7066463470458983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,12,12,128,1,fp8,fp8,0,0.07542719841003417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,12,1,128,1,float16,fp8,0,0.059592002630233766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,12,1,128,1,float16,fp8,0,0.9712688446044921
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,12,1,128,1,fp8,fp8,0,0.9711808204650879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,12,2,128,1,float16,float16,0,0.9927663803100586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,12,12,128,1,float16,float16,0,0.14974720478057862
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,12,2,128,1,fp8,fp8,0,0.9802559852600098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,12,2,128,1,float16,fp8,0,0.035764798521995544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,12,4,128,1,float16,float16,0,1.0658767700195313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,12,12,128,1,float16,float16,0,0.6350016117095947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,12,12,128,1,float16,fp8,0,0.5714416027069091
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,12,4,128,1,fp8,fp8,0,1.0186816215515138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,12,12,128,1,fp8,fp8,0,0.5895440101623535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,12,1,128,1,float16,float16,0,0.49839200973510744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,12,1,128,1,fp8,fp8,0,0.4961728096008301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,12,2,128,1,float16,fp8,0,0.5003344058990479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,12,2,128,1,fp8,fp8,0,0.5013872146606445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,12,4,128,1,float16,float16,0,0.5309567928314209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,12,4,128,1,float16,fp8,0,0.5166368007659912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,12,4,128,1,fp8,fp8,0,0.5150383949279785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,12,2,128,1,float16,fp8,0,0.9880880355834961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,12,4,128,1,float16,float16,0,0.03793280124664307
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,12,12,128,1,float16,fp8,0,0.2954832077026367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,12,12,128,1,fp8,fp8,0,0.29549920558929443
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,12,1,128,1,float16,float16,0,0.25590240955352783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,12,1,128,1,float16,fp8,0,0.25445919036865233
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,12,1,128,1,fp8,fp8,0,0.2544127941131592
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,12,4,128,1,float16,fp8,0,1.0082176208496094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,12,2,128,1,float16,float16,0,0.25978240966796873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,12,2,128,1,float16,fp8,0,0.2572736024856567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,12,2,128,1,fp8,fp8,0,0.2567296028137207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,12,4,128,1,float16,float16,0,0.2720623970031738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,12,4,128,1,float16,fp8,0,0.2624160051345825
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,12,12,128,1,float16,fp8,0,0.1544111967086792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,12,1,128,1,float16,fp8,0,0.4891664028167725
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,12,12,128,1,fp8,fp8,0,0.15351519584655762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,12,1,128,1,float16,float16,0,0.13386240005493164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,12,1,128,1,float16,fp8,0,0.13388160467147828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,12,2,128,1,float16,float16,0,0.5138095855712891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,12,1,128,1,fp8,fp8,0,0.13216480016708373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,12,2,128,1,float16,float16,0,0.1371503949165344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,12,2,128,1,float16,fp8,0,0.13379039764404296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,12,2,128,1,fp8,fp8,0,0.13472800254821776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,12,4,128,1,float16,float16,0,0.1447327971458435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,12,4,128,1,fp8,fp8,0,0.13878079652786254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,12,12,128,1,float16,float16,0,0.0958400011062622
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,12,12,128,1,fp8,fp8,0,0.08351519703865051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,12,1,128,1,float16,fp8,0,0.0699072003364563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,12,1,128,1,fp8,fp8,0,0.07035840153694153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,12,2,128,1,float16,float16,0,0.07339519858360291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,12,2,128,1,float16,fp8,0,0.0705456018447876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,12,2,128,1,fp8,fp8,0,0.06988800168037415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,12,4,128,1,float16,float16,0,0.0783631980419159
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,12,4,128,1,float16,fp8,0,0.07262719869613647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,12,4,128,1,fp8,fp8,0,0.07299360036849975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,12,12,128,1,float16,float16,0,0.059115201234817505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,12,12,128,1,fp8,fp8,0,0.045377600193023684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,12,1,128,1,float16,float16,0,0.04134399890899658
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,12,1,128,1,float16,fp8,0,0.0406495988368988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,12,1,128,1,fp8,fp8,0,0.04052639901638031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,12,2,128,1,float16,float16,0,0.041280001401901245
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,12,2,128,1,float16,fp8,0,0.04052639901638031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,12,2,128,1,fp8,fp8,0,0.04030559957027435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,12,4,128,1,float16,float16,0,0.04238399863243103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,12,4,128,1,float16,fp8,0,0.04103200137615204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,12,4,128,1,fp8,fp8,0,0.0409168004989624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,12,12,128,1,float16,float16,0,0.03266879916191101
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,12,12,128,1,float16,float16,0,0.1762928009033203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,12,12,128,1,float16,fp8,0,0.031867200136184694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,12,1,128,1,float16,float16,0,0.030428799986839294
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,12,1,128,1,float16,fp8,0,0.03038559854030609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,12,1,128,1,fp8,fp8,0,0.0302047997713089
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,12,2,128,1,float16,float16,0,0.03028959929943085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,12,2,128,1,float16,fp8,0,0.03010239899158478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,12,12,128,1,float16,fp8,0,0.08325440287590027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,12,1,128,1,float16,float16,0,0.07208160161972046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,12,4,128,1,float16,float16,0,0.031318399310112
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,12,4,128,1,float16,fp8,0,0.03070240020751953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,12,4,128,1,fp8,fp8,0,0.030980798602104186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,12,12,128,1,float16,fp8,0,0.04515359997749328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,12,12,128,1,float16,float16,0,0.33124480247497556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,12,1,128,1,float16,float16,0,1.1580559730529785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,12,1,128,1,float16,fp8,0,1.164571189880371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,12,1,128,1,fp8,fp8,0,1.1678432464599608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,12,4,128,1,fp8,fp8,0,0.26132800579071047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,12,12,128,1,fp8,fp8,0,0.032348799705505374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,12,4,128,1,float16,fp8,0,0.13762880563735963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,12,2,128,1,fp8,fp8,0,0.030195200443267824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,12,4,128,1,float16,float16,0,1.256612777709961
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,12,4,128,1,float16,fp8,0,1.2148271560668946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,12,4,128,1,fp8,fp8,0,1.2201343536376954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,12,12,128,1,fp8,fp8,0,0.7093760013580322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,12,1,128,1,float16,float16,0,0.5913280010223388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,12,1,128,1,float16,fp8,0,0.5895103931427002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,12,1,128,1,fp8,fp8,0,0.5910336017608643
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,12,2,128,1,float16,float16,0,1.1847408294677735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,12,2,128,1,float16,float16,0,0.6012351989746094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,12,2,128,1,float16,fp8,0,0.596072006225586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,12,2,128,1,float16,fp8,0,1.1703359603881835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,12,2,128,1,fp8,fp8,0,0.5964255809783936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,12,4,128,1,float16,float16,0,0.6379007816314697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,12,4,128,1,float16,fp8,0,0.616539192199707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,12,12,128,1,float16,float16,0,0.39388480186462405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,12,4,128,1,fp8,fp8,0,0.6194384098052979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,12,12,128,1,float16,fp8,0,0.7044303894042969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,12,12,128,1,fp8,fp8,0,0.35878241062164307
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,12,1,128,1,float16,float16,0,0.3041584014892578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,12,1,128,1,float16,fp8,0,0.3014591932296753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,12,1,128,1,fp8,fp8,0,0.30246241092681886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,12,2,128,1,float16,fp8,0,0.30484158992767335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,12,2,128,1,fp8,fp8,0,0.3053296089172363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,12,4,128,1,float16,float16,0,0.32460319995880127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,12,4,128,1,float16,fp8,0,0.3165760040283203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,12,4,128,1,fp8,fp8,0,0.31650240421295167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,12,12,128,1,float16,float16,0,0.2041696071624756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,12,12,128,1,float16,fp8,0,0.1878399968147278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,12,12,128,1,fp8,fp8,0,0.18789600133895873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,12,1,128,1,float16,float16,0,0.15903199911117555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,12,2,128,1,fp8,fp8,0,1.178054428100586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,12,1,128,1,float16,fp8,0,0.15834879875183105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,12,1,128,1,fp8,fp8,0,0.15782079696655274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,12,2,128,1,float16,float16,0,0.1622223973274231
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,12,2,128,1,fp8,fp8,0,0.15986080169677735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,12,2,128,1,float16,fp8,0,0.16112639904022216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,12,4,128,1,float16,float16,0,0.17069599628448487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,12,4,128,1,float16,fp8,0,0.16561280488967894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,12,12,128,1,float16,fp8,0,0.10149439573287963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,12,12,128,1,float16,float16,0,0.11007519960403442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,12,12,128,1,float16,float16,0,0.7662415981292725
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,12,1,128,1,float16,float16,0,0.08641600012779235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,12,1,128,1,float16,fp8,0,0.08428159952163697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,12,2,128,1,float16,float16,0,0.08851680159568787
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,12,2,128,1,float16,fp8,0,0.08653759956359863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,12,2,128,1,fp8,fp8,0,0.08635200262069702
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,12,4,128,1,float16,float16,0,0.09285280108451843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,12,4,128,1,float16,fp8,0,0.08999680280685425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,12,4,128,1,fp8,fp8,0,0.09058399796485901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,12,12,128,1,float16,float16,0,0.06127679944038391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,12,12,128,1,float16,fp8,0,0.056377601623535153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,12,1,128,1,float16,float16,0,0.04660159945487976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,12,1,128,1,float16,fp8,0,0.04710879921913147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,12,2,128,1,float16,float16,0,0.04687039852142334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,12,1,128,1,fp8,fp8,0,0.04688799977302551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,12,2,128,1,float16,fp8,0,0.0478767991065979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,12,2,128,1,fp8,fp8,0,0.04792320132255554
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,12,4,128,1,float16,float16,0,0.05113599896430969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,12,4,128,1,float16,fp8,0,0.04856959879398346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,12,4,128,1,fp8,fp8,0,0.04853439927101135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,12,12,128,1,float16,fp8,0,0.03232479989528656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,12,12,128,1,float16,float16,0,0.03300159871578216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,12,12,128,1,fp8,fp8,0,0.03237279951572418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,12,1,128,1,float16,float16,0,0.027820798754692077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,12,1,128,1,float16,fp8,0,0.028604799509048463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,12,1,128,1,fp8,fp8,0,0.028518399596214293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,12,2,128,1,float16,float16,0,0.028075200319290162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,12,2,128,1,float16,fp8,0,0.029993599653244017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,12,2,128,1,fp8,fp8,0,0.0301472008228302
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,12,4,128,1,float16,float16,0,0.028832000494003297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,12,4,128,1,fp8,fp8,0,0.16571680307388306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,12,4,128,1,fp8,fp8,0,0.0305759996175766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,12,12,128,1,fp8,fp8,0,0.10109440088272095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,12,12,128,1,float16,float16,0,0.03662239909172058
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,12,12,128,1,fp8,fp8,0,0.03788639903068543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,12,12,128,1,float16,fp8,0,0.3607055902481079
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,12,1,128,1,float16,fp8,0,0.031758400797843936
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,12,1,128,1,fp8,fp8,0,0.031579199433326724
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,12,1,128,1,fp8,fp8,0,0.08420640230178833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,12,2,128,1,float16,fp8,0,0.03354080021381378
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,12,2,128,1,fp8,fp8,0,0.033827200531959534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,12,2,128,1,float16,float16,0,0.3085088014602661
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,12,4,128,1,float16,fp8,0,0.03519999980926514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,12,4,128,1,fp8,fp8,0,0.03506560027599335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,12,12,128,1,fp8,fp8,0,0.05604959726333618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,12,1,128,1,float16,float16,0,0.719212818145752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,12,1,128,1,float16,fp8,0,0.740337610244751
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,12,1,128,1,fp8,fp8,0,0.7446400165557862
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,12,2,128,1,float16,float16,0,0.7412399768829345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,12,2,128,1,float16,fp8,0,0.7526095867156982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,12,2,128,1,fp8,fp8,0,0.7567232131958008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,12,4,128,1,float16,fp8,0,0.030555200576782227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,12,12,128,1,float16,fp8,0,0.037503999471664426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,12,4,128,1,float16,float16,0,0.8041279792785645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,12,4,128,1,float16,fp8,0,0.7983295917510986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,12,2,128,1,float16,float16,0,0.03325439989566803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,12,12,128,1,float16,fp8,0,0.5011280059814454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,12,4,128,1,float16,float16,0,0.034897598624229434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,12,1,128,1,float16,float16,0,0.3672544002532959
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,12,1,128,1,float16,fp8,0,0.3801664113998413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,12,1,128,1,fp8,fp8,0,0.38111679553985595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,12,2,128,1,float16,float16,0,0.3797055959701538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,12,2,128,1,float16,fp8,0,0.3849184036254883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,12,2,128,1,fp8,fp8,0,0.3868880033493042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,12,4,128,1,float16,float16,0,0.4130335807800293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,12,4,128,1,float16,fp8,0,0.4090303897857666
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,12,4,128,1,fp8,fp8,0,0.40786080360412597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,12,12,128,1,float16,fp8,0,0.25813119411468505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,12,12,128,1,fp8,fp8,0,0.2579583883285522
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,12,1,128,1,float16,fp8,0,0.19777120351791383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,12,1,128,1,fp8,fp8,0,0.19728319644927977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,12,2,128,1,float16,float16,0,0.19712799787521362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,12,2,128,1,float16,fp8,0,0.2008336067199707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,12,2,128,1,fp8,fp8,0,0.20064160823822022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,12,1,128,1,float16,float16,0,0.030265599489212036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,12,12,128,1,float16,float16,0,0.5482319831848145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,12,4,128,1,fp8,fp8,0,0.8022159576416016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,12,4,128,1,float16,fp8,0,0.21185119152069093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,12,12,128,1,float16,float16,0,0.1486143946647644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,12,12,128,1,float16,fp8,0,0.13613760471343994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,12,1,128,1,float16,float16,0,0.10268160104751586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,12,1,128,1,float16,fp8,0,0.1051535964012146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,12,1,128,1,fp8,fp8,0,0.10523359775543213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,12,2,128,1,float16,float16,0,0.10556479692459106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,12,12,128,1,float16,float16,0,0.281279993057251
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,12,2,128,1,fp8,fp8,0,0.10679199695587158
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,12,1,128,1,float16,float16,0,0.19099040031433107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,12,4,128,1,float16,float16,0,0.11340320110321045
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,12,4,128,1,float16,fp8,0,0.11286720037460327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,12,4,128,1,fp8,fp8,0,0.11233760118484497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,12,12,128,1,float16,fp8,0,0.0748799979686737
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,12,12,128,1,fp8,fp8,0,0.07525759935379028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,12,1,128,1,float16,float16,0,0.0581055998802185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,12,1,128,1,float16,fp8,0,0.05739359855651856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,12,1,128,1,fp8,fp8,0,0.057436800003051756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,12,2,128,1,float16,float16,0,0.05940480232238769
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,12,2,128,1,float16,fp8,0,0.05967680215835571
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,12,2,128,1,fp8,fp8,0,0.059566402435302736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,12,4,128,1,float16,float16,0,0.06428639888763428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,12,4,128,1,fp8,fp8,0,0.2127903938293457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,12,4,128,1,fp8,fp8,0,0.06285279989242554
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,12,12,128,1,float16,float16,0,0.04766719937324524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,12,12,128,1,fp8,fp8,0,0.13576480150222778
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,12,12,128,1,float16,fp8,0,0.04239999949932098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,12,12,128,1,fp8,fp8,0,0.042401599884033206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,12,1,128,1,float16,float16,0,0.032846400141716005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,12,1,128,1,float16,fp8,0,0.03471199870109558
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,12,2,128,1,float16,float16,0,0.03289600014686585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,12,2,128,1,float16,fp8,0,0.035183998942375186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,12,2,128,1,fp8,fp8,0,0.03530080020427704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,12,4,128,1,float16,float16,0,0.03596160113811493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,12,4,128,1,float16,fp8,0,0.03585599958896637
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,12,4,128,1,fp8,fp8,0,0.035846400260925296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,12,12,128,1,float16,float16,0,0.02666719853878021
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,12,12,128,1,float16,fp8,0,0.026332798600196838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,12,2,128,1,float16,fp8,0,0.10696320533752442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,12,1,128,1,float16,float16,0,0.022232000529766083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,12,1,128,1,float16,fp8,0,0.02410400062799454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,12,1,128,1,fp8,fp8,0,0.024276800453662872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,12,2,128,1,float16,float16,0,0.022793599963188173
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,12,2,128,1,float16,fp8,0,0.024219200015068054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,12,2,128,1,fp8,fp8,0,0.024326400458812715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,12,4,128,1,float16,float16,0,0.023185600340366364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,12,4,128,1,float16,fp8,0,0.024827200174331664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,12,4,128,1,fp8,fp8,0,0.024803200364112855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,12,12,128,1,float16,float16,0,0.0196383997797966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,12,12,128,1,float16,float16,0,0.08121920228004456
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,12,12,128,1,fp8,fp8,0,0.020204800367355346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,12,1,128,1,float16,float16,0,0.017615999281406402
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,12,1,128,1,float16,fp8,0,0.019123199582099914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,12,1,128,1,fp8,fp8,0,0.019079999625682832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,12,2,128,1,float16,fp8,0,0.0189968004822731
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,12,2,128,1,fp8,fp8,0,0.0189423993229866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,12,4,128,1,float16,float16,0,0.018049600720405578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,12,4,128,1,float16,fp8,0,0.01926559954881668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,12,4,128,1,fp8,fp8,0,0.019551999866962433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,12,12,128,1,float16,float16,0,0.01849119961261749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,12,12,128,1,float16,fp8,0,0.01971199959516525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,12,4,128,1,float16,float16,0,0.21395680904388428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,12,12,128,1,fp8,fp8,0,0.019670400023460387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,12,1,128,1,float16,float16,0,0.017377600073814392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,12,1,128,1,float16,fp8,0,0.01884479969739914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,12,12,128,1,fp8,fp8,0,0.49817600250244143
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,12,2,128,1,float16,float16,0,0.017507199943065644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,12,2,128,1,fp8,fp8,0,0.019006399810314177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,12,2,128,1,float16,fp8,0,0.018889600038528444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,12,4,128,1,float16,float16,0,0.0176704004406929
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,12,4,128,1,float16,fp8,0,0.018822400271892546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,12,4,128,1,fp8,fp8,0,0.019011199474334717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,12,1,128,1,fp8,fp8,0,0.035011199116706845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,12,1,128,1,float16,float16,0,0.29875199794769286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,12,1,128,1,float16,fp8,0,0.31626238822937014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,12,1,128,1,fp8,fp8,0,0.31776158809661864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,12,12,128,1,fp8,fp8,0,0.02635999917984009
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,12,2,128,1,float16,float16,0,0.3099087953567505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,12,2,128,1,float16,fp8,0,0.32246720790863037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,12,2,128,1,fp8,fp8,0,0.324947190284729
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,12,4,128,1,float16,float16,0,0.3440608024597168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,12,4,128,1,float16,fp8,0,0.34506399631500245
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,12,12,128,1,float16,fp8,0,0.020283199846744537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,12,1,128,1,float16,float16,0,0.1579584002494812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,12,12,128,1,fp8,fp8,0,0.21551520824432374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,12,1,128,1,float16,fp8,0,0.1657711982727051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,12,1,128,1,fp8,fp8,0,0.1651263952255249
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,12,2,128,1,float16,float16,0,0.16347359418869017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,12,2,128,1,float16,fp8,0,0.16858559846878052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,12,2,128,1,fp8,fp8,0,0.16928160190582275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,12,4,128,1,float16,float16,0,0.17970399856567382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,12,4,128,1,float16,fp8,0,0.18086559772491456
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,12,1,128,1,fp8,fp8,0,0.01879040002822876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,12,12,128,1,float16,float16,0,0.13248000144958497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,12,12,128,1,float16,fp8,0,0.11488319635391235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,12,12,128,1,fp8,fp8,0,0.11479840278625489
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,12,1,128,1,float16,float16,0,0.08607199788093567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,12,1,128,1,float16,fp8,0,0.08962079882621765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,12,1,128,1,fp8,fp8,0,0.08957759737968445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,12,2,128,1,float16,float16,0,0.08936160206794738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,12,2,128,1,float16,fp8,0,0.09137920141220093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,12,12,128,1,float16,float16,0,0.24934399127960205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,12,12,128,1,float16,fp8,0,0.21604158878326415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,12,2,128,1,fp8,fp8,0,0.09136160016059876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,12,4,128,1,float16,float16,0,0.09711840152740478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,12,4,128,1,float16,fp8,0,0.09727360010147094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,12,2,128,1,float16,float16,0,0.017798399925231932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,12,4,128,1,fp8,fp8,0,0.09726079702377319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,12,12,128,1,float16,float16,0,0.07372639775276184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,12,12,128,1,float16,fp8,0,0.06462879776954651
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,12,12,128,1,fp8,fp8,0,0.06400160193443298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,12,1,128,1,fp8,fp8,0,0.049193599820137025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,12,2,128,1,float16,float16,0,0.05113599896430969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,12,2,128,1,float16,fp8,0,0.050361597537994386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,12,2,128,1,fp8,fp8,0,0.05092160105705261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,12,4,128,1,float16,float16,0,0.055731201171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,12,4,128,1,float16,fp8,0,0.054230397939682005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,12,4,128,1,fp8,fp8,0,0.05457119941711426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,12,12,128,1,float16,float16,0,0.04232639968395233
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,12,12,128,1,float16,fp8,0,0.03661440014839172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,12,12,128,1,fp8,fp8,0,0.036111998558044436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,12,4,128,1,fp8,fp8,0,0.179639995098114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,12,1,128,1,fp8,fp8,0,0.028508800268173217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,12,2,128,1,float16,float16,0,0.027508801221847533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,12,2,128,1,float16,fp8,0,0.02868959903717041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,12,2,128,1,fp8,fp8,0,0.02879360020160675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,12,4,128,1,float16,float16,0,0.030516800284385682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,12,4,128,1,float16,fp8,0,0.02999039888381958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,12,4,128,1,fp8,fp8,0,0.030364799499511718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,12,12,128,1,float16,float16,0,0.023579199612140656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,12,12,128,1,float16,fp8,0,0.022521600127220154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,12,12,128,1,fp8,fp8,0,0.02263679951429367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,12,1,128,1,float16,float16,0,0.019457599520683287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,12,1,128,1,float16,fp8,0,0.02080959975719452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,12,1,128,1,fp8,fp8,0,0.02073120027780533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,12,4,128,1,fp8,fp8,0,0.345688009262085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,12,2,128,1,float16,float16,0,0.01929759979248047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,12,2,128,1,float16,fp8,0,0.02094399929046631
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,12,4,128,1,float16,float16,0,0.020230400562286376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,12,4,128,1,float16,fp8,0,0.02133920043706894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,12,1,128,1,float16,float16,0,0.04975839853286743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,12,4,128,1,fp8,fp8,0,0.02134400010108948
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,12,12,128,1,float16,float16,0,0.016547200083732606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,12,12,128,1,float16,fp8,0,0.01648319959640503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,12,12,128,1,fp8,fp8,0,0.01637279987335205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,12,1,128,1,float16,float16,0,0.014591999351978302
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,12,1,128,1,fp8,fp8,0,0.01531360000371933
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,12,2,128,1,float16,float16,0,0.014716799557209014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,12,2,128,1,float16,fp8,0,0.015204800665378571
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,12,1,128,1,float16,float16,0,0.027718400955200194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,12,2,128,1,fp8,fp8,0,0.015340800583362579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,12,4,128,1,float16,float16,0,0.015121600031852723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,12,4,128,1,float16,fp8,0,0.06291519999504089
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,12,4,128,1,fp8,fp8,0,0.01541920006275177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,12,12,128,1,float16,float16,0,0.015083199739456177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,12,12,128,1,float16,fp8,0,0.015859200060367583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,12,1,128,1,float16,float16,0,0.014060799777507783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,12,1,128,1,float16,fp8,0,0.015097600221633912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,12,2,128,1,float16,float16,0,0.014350399374961853
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,12,2,128,1,float16,fp8,0,0.015065599977970124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,12,2,128,1,fp8,fp8,0,0.015065599977970124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,12,4,128,1,float16,float16,0,0.014407999813556671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,12,4,128,1,float16,fp8,0,0.015204800665378571
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,12,4,128,1,fp8,fp8,0,0.015223999321460725
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,12,12,128,1,float16,float16,0,0.014668799936771393
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,12,12,128,1,float16,fp8,0,0.015641599893569946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,12,12,128,1,fp8,fp8,0,0.015627199411392213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,12,1,128,1,float16,float16,0,0.013935999572277069
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,12,1,128,1,float16,fp8,0,0.014905600249767304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,12,1,128,1,fp8,fp8,0,0.014929600059986115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,12,2,128,1,float16,float16,0,0.01383039951324463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,12,2,128,1,fp8,fp8,0,0.020996800065040587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,12,2,128,1,fp8,fp8,0,0.01525759994983673
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,12,4,128,1,float16,float16,0,0.014188799262046813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,12,1,128,1,float16,fp8,0,0.04907520115375519
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,12,4,128,1,float16,fp8,0,0.015020799636840821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,12,4,128,1,fp8,fp8,0,0.015220800042152404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,12,1,128,1,float16,float16,0,0.14661439657211303
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,12,1,128,1,float16,fp8,0,0.15977760553359985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,12,1,128,1,fp8,fp8,0,0.1588544011116028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,12,2,128,1,float16,float16,0,0.15531680583953858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,12,2,128,1,float16,fp8,0,0.1623568058013916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,12,1,128,1,float16,fp8,0,0.028537601232528687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,12,2,128,1,fp8,fp8,0,0.16229599714279175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,12,12,128,1,fp8,fp8,0,0.015872000157833098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,12,4,128,1,float16,float16,0,0.16971679925918579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,12,1,128,1,fp8,fp8,0,0.015080000460147857
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,12,4,128,1,float16,fp8,0,0.1737920045852661
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,12,12,128,1,float16,float16,0,0.1322208046913147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,12,1,128,1,float16,float16,0,0.08243680000305176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,12,1,128,1,float16,fp8,0,0.08485599756240844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,12,1,128,1,fp8,fp8,0,0.08505920171737671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,12,2,128,1,float16,float16,0,0.08492159843444824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,12,2,128,1,float16,fp8,0,0.08616960048675537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,12,2,128,1,fp8,fp8,0,0.08656960129737853
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,12,4,128,1,float16,float16,0,0.09363840222358703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,12,4,128,1,float16,fp8,0,0.09194719791412354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,12,4,128,1,fp8,fp8,0,0.09244959950447082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,12,2,128,1,float16,fp8,0,0.014860799908638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,12,12,128,1,float16,fp8,0,0.06051679849624634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,12,12,128,1,fp8,fp8,0,0.06100000143051147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,12,1,128,1,float16,float16,0,0.046803200244903566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,12,1,128,1,float16,fp8,0,0.015300799906253815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,12,1,128,1,float16,fp8,0,0.04411199986934662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,12,1,128,1,fp8,fp8,0,0.04452320039272308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,12,2,128,1,float16,float16,0,0.0487311989068985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,12,2,128,1,float16,fp8,0,0.04604319930076599
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,12,4,128,1,float16,float16,0,0.052553600072860716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,12,4,128,1,float16,fp8,0,0.015622399747371674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,12,4,128,1,float16,fp8,0,0.04974240064620972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,12,4,128,1,fp8,fp8,0,0.04955199956893921
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,12,12,128,1,float16,float16,0,0.040454399585723874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,12,12,128,1,float16,fp8,0,0.03103039860725403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,12,4,128,1,fp8,fp8,0,0.17506879568099976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,12,12,128,1,fp8,fp8,0,0.031732800602912906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,12,1,128,1,float16,float16,0,0.02401919960975647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,12,12,128,1,float16,fp8,0,0.1137935996055603
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,12,1,128,1,fp8,fp8,0,0.02547520101070404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,12,12,128,1,fp8,fp8,0,0.11374560594558716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,12,2,128,1,float16,fp8,0,0.0254287987947464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,12,4,128,1,float16,float16,0,0.02701280117034912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,12,4,128,1,float16,fp8,0,0.025591999292373657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,12,4,128,1,fp8,fp8,0,0.025627198815345763
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,12,12,128,1,float16,float16,0,0.022040000557899474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,12,12,128,1,float16,fp8,0,0.020231999456882477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,12,12,128,1,fp8,fp8,0,0.02048480063676834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,12,1,128,1,float16,float16,0,0.017710399627685548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,12,1,128,1,float16,fp8,0,0.01889120042324066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,12,1,128,1,fp8,fp8,0,0.018863999843597413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,12,2,128,1,float16,float16,0,0.018001599609851836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,12,2,128,1,float16,fp8,0,0.018831999599933626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,12,2,128,1,fp8,fp8,0,0.01886080056428909
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,12,4,128,1,float16,float16,0,0.01847199946641922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,12,4,128,1,float16,fp8,0,0.01911199986934662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,12,4,128,1,fp8,fp8,0,0.018913599848747253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,12,12,128,1,float16,float16,0,0.014803199470043183
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,12,12,128,1,float16,fp8,0,0.01422560065984726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,12,12,128,1,fp8,fp8,0,0.014521600306034088
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,12,1,128,1,float16,float16,0,0.013009600341320038
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,12,1,128,1,float16,fp8,0,0.013788799941539764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,12,1,128,1,fp8,fp8,0,0.013457599282264709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,12,2,128,1,float16,float16,0,0.013337600231170654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,12,2,128,1,float16,fp8,0,0.013844799995422364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,12,2,128,1,fp8,fp8,0,0.013798399269580841
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,12,4,128,1,float16,float16,0,0.013491199910640716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,12,4,128,1,float16,fp8,0,0.01390720009803772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,12,4,128,1,fp8,fp8,0,0.013769599795341491
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,12,12,128,1,float16,float16,0,0.013691200315952301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,12,12,128,1,float16,fp8,0,0.013724799454212188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,12,12,128,1,fp8,fp8,0,0.013748799264431
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,12,1,128,1,float16,float16,0,0.012449599802494049
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,12,1,128,1,float16,fp8,0,0.013398399949073792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,12,1,128,1,fp8,fp8,0,0.01305759996175766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,12,2,128,1,float16,float16,0,0.012700800597667695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,12,2,128,1,float16,fp8,0,0.013222399353981017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,12,2,128,1,fp8,fp8,0,0.013279999792575835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,12,4,128,1,float16,float16,0,0.012972800433635712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,12,4,128,1,float16,fp8,0,0.013286399841308593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,12,2,128,1,fp8,fp8,0,0.04596959948539734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,12,12,128,1,float16,float16,0,0.01313920021057129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,12,12,128,1,float16,fp8,0,0.013388800621032714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,12,1,128,1,float16,float16,0,0.012326399981975555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,12,1,128,1,float16,fp8,0,0.01300320029258728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,12,1,128,1,fp8,fp8,0,0.013124799728393555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,12,2,128,1,float16,float16,0,0.01244639977812767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,12,1,128,1,float16,fp8,0,0.025326400995254517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,12,2,128,1,float16,float16,0,0.02444159984588623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,12,2,128,1,fp8,fp8,0,0.013174399733543396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,12,4,128,1,float16,float16,0,0.01266240030527115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,12,4,128,1,float16,fp8,0,0.013214400410652161
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,12,4,128,1,fp8,fp8,0,0.01302880048751831
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,12,12,128,1,float16,float16,0,0.012775999307632447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,12,12,128,1,float16,fp8,0,0.01311360001564026
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,12,12,128,1,fp8,fp8,0,0.013361600041389466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,12,1,128,1,float16,fp8,0,0.012595200538635254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,12,1,128,1,fp8,fp8,0,0.01273760050535202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,12,2,128,1,float16,float16,0,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,12,2,128,1,float16,fp8,0,0.012772800028324127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,12,2,128,1,fp8,fp8,0,0.012608000636100769
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,12,4,128,1,float16,float16,0,0.012139199674129486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,12,4,128,1,float16,fp8,0,0.01279519945383072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,12,4,128,1,fp8,fp8,0,0.01297920048236847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,12,1,128,1,float16,float16,0,0.0810800015926361
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,12,1,128,1,float16,fp8,0,0.0863856017589569
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,12,1,128,1,fp8,fp8,0,0.08613920211791992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,12,2,128,1,float16,float16,0,0.08597279787063598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,12,2,128,1,float16,fp8,0,0.08675839900970458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,12,2,128,1,fp8,fp8,0,0.08744320273399353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,12,4,128,1,float16,float16,0,0.10578720569610596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,12,4,128,1,float16,fp8,0,0.10917279720306397
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,12,4,128,1,fp8,fp8,0,0.10870239734649659
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,12,12,128,1,float16,fp8,0,0.08783519864082337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,12,12,128,1,fp8,fp8,0,0.08770400285720825
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,12,4,128,1,fp8,fp8,0,0.013361600041389466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,12,1,128,1,float16,float16,0,0.04747999906539917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,12,1,128,1,float16,fp8,0,0.045475199818611145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,12,1,128,1,fp8,fp8,0,0.04544639885425568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,12,2,128,1,float16,float16,0,0.04917599856853485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,12,2,128,1,float16,fp8,0,0.046158400177955625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,12,2,128,1,float16,fp8,0,0.013022400438785553
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,12,2,128,1,fp8,fp8,0,0.025542399287223815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,12,4,128,1,float16,float16,0,0.06049759984016419
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,12,4,128,1,float16,fp8,0,0.058955198526382445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,12,4,128,1,fp8,fp8,0,0.05870720148086548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,12,1,128,1,float16,float16,0,0.012201599776744843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,12,12,128,1,float16,float16,0,0.05221279859542847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,12,12,128,1,float16,fp8,0,0.046244800090789795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,12,12,128,1,fp8,fp8,0,0.04609119892120361
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,12,1,128,1,float16,float16,0,0.02431199997663498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,12,1,128,1,float16,fp8,0,0.025536000728607178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,12,1,128,1,fp8,fp8,0,0.025947201251983642
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,12,2,128,1,float16,float16,0,0.02439039945602417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,12,2,128,1,float16,fp8,0,0.025595200061798096
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,12,2,128,1,fp8,fp8,0,0.025900799036026
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,12,4,128,1,float16,float16,0,0.031672000885009766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,12,4,128,1,float16,fp8,0,0.02991360127925873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,12,4,128,1,fp8,fp8,0,0.030137598514556885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,12,12,128,1,float16,float16,0,0.025830399990081788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,12,12,128,1,fp8,fp8,0,0.024803200364112855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,12,1,128,1,float16,fp8,0,0.019206400215625762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,12,12,128,1,float16,float16,0,0.07233279943466187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,12,2,128,1,float16,float16,0,0.01825760006904602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,12,2,128,1,float16,fp8,0,0.019254399836063384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,12,12,128,1,float16,float16,0,0.0937391996383667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,12,4,128,1,float16,float16,0,0.01852799952030182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,12,4,128,1,float16,fp8,0,0.01932799965143204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,12,4,128,1,fp8,fp8,0,0.019339199364185333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,12,12,128,1,float16,float16,0,0.017953599989414214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,12,12,128,1,float16,fp8,0,0.018662400543689728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,12,12,128,1,fp8,fp8,0,0.013729600608348847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,12,1,128,1,float16,float16,0,0.013276800513267517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,12,1,128,1,float16,fp8,0,0.013729600608348847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,12,1,128,1,fp8,fp8,0,0.013876800239086152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,12,2,128,1,float16,float16,0,0.013310399651527405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,12,2,128,1,float16,fp8,0,0.013579200208187103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,12,2,128,1,fp8,fp8,0,0.04699679911136627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,12,2,128,1,fp8,fp8,0,0.013784000277519226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,12,4,128,1,float16,float16,0,0.013318400084972381
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,12,4,128,1,float16,fp8,0,0.013767999410629273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,12,4,128,1,fp8,fp8,0,0.013619199395179749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,12,12,128,1,float16,float16,0,0.013308799266815186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,12,12,128,1,float16,fp8,0,0.013956800103187561
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,12,12,128,1,fp8,fp8,0,0.013752000033855438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,12,1,128,1,float16,float16,0,0.012724800407886505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,12,1,128,1,float16,fp8,0,0.013673600554466248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,12,1,128,1,fp8,fp8,0,0.013355199992656708
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,12,2,128,1,float16,float16,0,0.013115200400352477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,12,2,128,1,float16,fp8,0,0.013220800459384919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,12,2,128,1,fp8,fp8,0,0.01353600025177002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,12,4,128,1,float16,float16,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,12,4,128,1,float16,fp8,0,0.013415999710559845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,12,12,128,1,float16,fp8,0,0.024751999974250795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,12,4,128,1,fp8,fp8,0,0.013513599336147309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,12,12,128,1,float16,float16,0,0.01276959925889969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,12,1,128,1,float16,float16,0,0.018059200048446654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,12,12,128,1,float16,fp8,0,0.013348799943923951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,12,1,128,1,float16,float16,0,0.012377600371837615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,12,1,128,1,float16,fp8,0,0.01324159950017929
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,12,1,128,1,fp8,fp8,0,0.013099199533462525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,12,1,128,1,fp8,fp8,0,0.019196799397468566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,12,2,128,1,float16,fp8,0,0.013374400138854981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,12,4,128,1,float16,fp8,0,0.013089600205421447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,12,4,128,1,fp8,fp8,0,0.013104000687599182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,12,12,128,1,float16,fp8,0,0.013284799456596375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,12,12,128,1,fp8,fp8,0,0.013305599987506866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,12,1,128,1,float16,float16,0,0.012009599804878235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,12,1,128,1,float16,fp8,0,0.012814399600028992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,12,1,128,1,fp8,fp8,0,0.012849600613117218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,12,2,128,1,float16,float16,0,0.01226240023970604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,12,2,128,1,float16,fp8,0,0.012891200184822083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,12,2,128,1,fp8,fp8,0,0.012880000472068786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,12,12,128,1,fp8,fp8,0,0.01867839992046356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,12,4,128,1,float16,float16,0,0.01231200024485588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,12,4,128,1,fp8,fp8,0,0.012889599800109864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,12,12,128,1,float16,float16,0,0.012265600264072418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,12,12,128,1,float16,fp8,0,0.013120000064373017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,12,12,128,1,fp8,fp8,0,0.01311040073633194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,12,1,128,1,float16,float16,0,0.012012799829244613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,12,1,128,1,float16,fp8,0,0.012716799974441528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,12,1,128,1,fp8,fp8,0,0.012755200266838074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,12,2,128,1,float16,float16,0,0.012116800248622894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,12,2,128,1,float16,fp8,0,0.01255200058221817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,12,2,128,1,fp8,fp8,0,0.012555199861526489
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,12,4,128,1,float16,float16,0,0.01194240003824234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,12,4,128,1,float16,fp8,0,0.012824000418186187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,12,4,128,1,fp8,fp8,0,0.012827199697494508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,12,1,128,1,float16,float16,0,0.048019200563430786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,12,1,128,1,float16,fp8,0,0.047075200080871585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,12,12,128,1,fp8,fp8,0,0.013470399379730224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,12,1,128,1,fp8,fp8,0,0.04723680019378662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,12,2,128,1,float16,float16,0,0.012436799705028534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,12,2,128,1,float16,float16,0,0.057339197397232054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,12,2,128,1,fp8,fp8,0,0.05717440247535706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,12,4,128,1,float16,float16,0,0.061896002292633055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,12,12,128,1,float16,float16,0,0.012432000041007996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,12,4,128,1,float16,fp8,0,0.06046559810638428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,12,4,128,1,fp8,fp8,0,0.06021440029144287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,12,12,128,1,float16,float16,0,0.0749776005744934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,12,12,128,1,float16,fp8,0,0.07396320104599
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,12,12,128,1,fp8,fp8,0,0.0740112006664276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,12,1,128,1,float16,fp8,0,0.026686400175094604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,12,1,128,1,fp8,fp8,0,0.026675200462341307
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,12,2,128,1,float16,float16,0,0.028446400165557863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,12,2,128,1,float16,fp8,0,0.031057599186897277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,12,4,128,1,float16,fp8,0,0.012984000146389008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,12,2,128,1,fp8,fp8,0,0.031240001320838928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,12,4,128,1,float16,float16,0,0.031692799925804135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,12,4,128,1,fp8,fp8,0,0.031384000182151796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,12,12,128,1,float16,float16,0,0.03682880103588104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,12,12,128,1,float16,fp8,0,0.038899201154708865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,12,12,128,1,fp8,fp8,0,0.038936001062393186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,12,1,128,1,float16,float16,0,0.01842080056667328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,12,1,128,1,float16,fp8,0,0.019912000000476836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,12,1,128,1,fp8,fp8,0,0.019964799284934998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,12,2,128,1,float16,float16,0,0.01836480051279068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,12,2,128,1,float16,fp8,0,0.019596800208091736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,12,2,128,1,fp8,fp8,0,0.019652800261974336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,12,4,128,1,float16,float16,0,0.018588800728321076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,12,4,128,1,float16,fp8,0,0.019761599600315094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,12,4,128,1,fp8,fp8,0,0.019758400321006776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,12,2,128,1,fp8,fp8,0,0.013100799918174744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,12,12,128,1,float16,float16,0,0.022299200296401978
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,12,12,128,1,float16,fp8,0,0.02354079931974411
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,12,12,128,1,fp8,fp8,0,0.023691199719905853
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,12,1,128,1,float16,float16,0,0.01308639943599701
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,12,1,128,1,float16,fp8,0,0.01417119950056076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,12,1,128,1,fp8,fp8,0,0.013980799913406372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,12,2,128,1,float16,fp8,0,0.014057600498199463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,12,2,128,1,fp8,fp8,0,0.014056000113487243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,12,1,128,1,float16,float16,0,0.024905599653720856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,12,4,128,1,float16,float16,0,0.013540799915790557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,12,4,128,1,float16,fp8,0,0.014291200041770934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,12,4,128,1,fp8,fp8,0,0.014129599928855896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,12,12,128,1,float16,float16,0,0.017220799624919892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,12,12,128,1,float16,fp8,0,0.01812320053577423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,12,12,128,1,fp8,fp8,0,0.018225599825382233
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,12,1,128,1,float16,float16,0,0.01247519999742508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,12,1,128,1,float16,fp8,0,0.013633599877357483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,12,1,128,1,fp8,fp8,0,0.013193599879741669
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,12,2,128,1,float16,float16,0,0.012803199887275695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,12,2,128,1,float16,fp8,0,0.013084800541400909
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,12,2,128,1,fp8,fp8,0,0.013412800431251527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,12,4,128,1,float16,float16,0,0.012464000284671784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,12,4,128,1,float16,fp8,0,0.013382400572299957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,12,4,128,1,fp8,fp8,0,0.013104000687599182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,12,12,128,1,float16,float16,0,0.01281919926404953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,12,12,128,1,float16,fp8,0,0.013439999520778656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,12,12,128,1,fp8,fp8,0,0.013447999954223633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,12,1,128,1,float16,float16,0,0.01252480000257492
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,12,1,128,1,float16,fp8,0,0.013222399353981017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,12,1,128,1,fp8,fp8,0,0.01313920021057129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,12,2,128,1,float16,float16,0,0.012555199861526489
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,12,2,128,1,float16,fp8,0,0.013076800107955932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,12,2,128,1,fp8,fp8,0,0.01316159963607788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,12,4,128,1,float16,float16,0,0.012244799733161926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,12,4,128,1,float16,fp8,0,0.013299199938774108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,12,4,128,1,fp8,fp8,0,0.013012799620628356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,12,12,128,1,float16,float16,0,0.012561599910259246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,12,12,128,1,float16,fp8,0,0.013219200074672699
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,12,12,128,1,fp8,fp8,0,0.013391999900341034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,12,1,128,1,float16,float16,0,0.011910399794578553
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,12,1,128,1,float16,fp8,0,0.012692800164222718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,12,1,128,1,fp8,fp8,0,0.012625600397586822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,12,2,128,1,float16,float16,0,0.011881600320339202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,12,2,128,1,float16,fp8,0,0.012611199915409089
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,12,2,128,1,fp8,fp8,0,0.012667199969291687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,12,4,128,1,float16,float16,0,0.011886399984359742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,12,4,128,1,float16,fp8,0,0.012731200456619263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,12,4,128,1,fp8,fp8,0,0.01273919939994812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,12,12,128,1,float16,fp8,0,0.01300159990787506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,12,12,128,1,fp8,fp8,0,0.01319040060043335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,12,1,128,1,float16,float16,0,0.011767999827861786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,12,1,128,1,float16,fp8,0,0.01268800050020218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,12,1,128,1,fp8,fp8,0,0.012697599828243256
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,12,2,128,1,float16,float16,0,0.012019199877977371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,12,2,128,1,float16,fp8,0,0.012664000689983367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,12,2,128,1,float16,fp8,0,0.05749599933624268
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,12,4,128,1,float16,float16,0,0.011795199662446975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,12,4,128,1,float16,float16,0,0.012404800206422806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,12,4,128,1,float16,fp8,0,0.012537600100040435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,12,12,128,1,float16,float16,0,0.012060800194740295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,12,2,128,1,fp8,fp8,0,0.019270400702953338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,12,2,128,1,float16,float16,0,0.013406400382518769
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,12,1,128,1,float16,float16,0,0.011952000111341477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,12,1,128,1,float16,fp8,0,0.012726399302482604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,12,2,128,1,float16,float16,0,0.01196959987282753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,12,2,128,1,float16,fp8,0,0.012587200105190276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,12,2,128,1,fp8,fp8,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,12,4,128,1,float16,float16,0,0.011823999881744384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,12,4,128,1,float16,fp8,0,0.012577599287033081
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,12,4,128,1,fp8,fp8,0,0.01260959953069687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,12,1,128,1,float16,float16,0,0.03003840148448944
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,12,4,128,1,float16,fp8,0,0.031251201033592226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,12,12,128,1,float16,float16,0,0.012548799812793731
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,12,1,128,1,fp8,fp8,0,0.03374080061912536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,12,2,128,1,float16,float16,0,0.03054719865322113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,12,2,128,1,float16,fp8,0,0.033590400218963624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,12,12,128,1,float16,fp8,0,0.012918399274349212
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,12,12,128,1,fp8,fp8,0,0.012788799405097962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,12,4,128,1,float16,float16,0,0.04758400022983551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,12,4,128,1,float16,fp8,0,0.050723201036453246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,12,1,128,1,fp8,fp8,0,0.0126351997256279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,12,4,128,1,fp8,fp8,0,0.0512719988822937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,12,12,128,1,float16,float16,0,0.05941759943962097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,12,12,128,1,float16,fp8,0,0.06688320040702819
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,12,1,128,1,float16,fp8,0,0.02093919962644577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,12,1,128,1,fp8,fp8,0,0.020950399339199066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,12,2,128,1,float16,float16,0,0.01937599927186966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,12,2,128,1,float16,fp8,0,0.020716799795627593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,12,2,128,1,fp8,fp8,0,0.02072640061378479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,12,1,128,1,float16,fp8,0,0.03328480124473572
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,12,4,128,1,float16,float16,0,0.02709600031375885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,12,4,128,1,float16,fp8,0,0.029927998781204224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,12,4,128,1,fp8,fp8,0,0.029791998863220214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,12,12,128,1,float16,float16,0,0.03363839983940124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,12,12,128,1,float16,fp8,0,0.03772799968719483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,12,12,128,1,fp8,fp8,0,0.037724798917770384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,12,1,128,1,float16,float16,0,0.013945600390434265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,12,1,128,1,float16,fp8,0,0.014777599275112152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,12,1,128,1,fp8,fp8,0,0.014764800667762756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,12,2,128,1,float16,float16,0,0.013923199474811554
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,12,2,128,1,fp8,fp8,0,0.014617599546909332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,12,4,128,1,float16,float16,0,0.017956799268722533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,12,4,128,1,float16,fp8,0,0.019246399402618408
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,12,4,128,1,fp8,fp8,0,0.019204799830913544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,12,12,128,1,float16,float16,0,0.021299199759960176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,12,12,128,1,float16,fp8,0,0.023175999522209167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,12,12,128,1,fp8,fp8,0,0.02343519926071167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,12,1,128,1,float16,float16,0,0.012990400195121765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,12,1,128,1,float16,fp8,0,0.01387840062379837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,12,2,128,1,fp8,fp8,0,0.012639999389648438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,12,2,128,1,float16,float16,0,0.012904000282287598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,12,2,128,1,float16,fp8,0,0.013593600690364837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,12,2,128,1,fp8,fp8,0,0.013638399541378021
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,12,2,128,1,fp8,fp8,0,0.03358240127563476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,12,4,128,1,float16,fp8,0,0.013670399785041809
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,12,4,128,1,fp8,fp8,0,0.013844799995422364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,12,12,128,1,float16,float16,0,0.016577599942684172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,12,12,128,1,float16,fp8,0,0.01775519996881485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,12,1,128,1,float16,float16,0,0.012470400333404541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,12,12,128,1,fp8,fp8,0,0.06761599779129028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,12,1,128,1,fp8,fp8,0,0.013531200587749481
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,12,2,128,1,float16,float16,0,0.012668800354003907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,12,2,128,1,float16,fp8,0,0.013091200590133667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,12,2,128,1,fp8,fp8,0,0.013412800431251527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,12,4,128,1,float16,float16,0,0.012561599910259246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,12,4,128,1,float16,fp8,0,0.013115200400352477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,12,4,128,1,fp8,fp8,0,0.013211199641227722
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,12,12,128,1,float16,float16,0,0.012744000554084778
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,12,12,128,1,float16,fp8,0,0.013625599443912506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,12,12,128,1,fp8,fp8,0,0.013478399813175201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,12,2,128,1,float16,fp8,0,0.014691199362277984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,12,1,128,1,float16,fp8,0,0.013131199777126313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,12,1,128,1,fp8,fp8,0,0.013140800595283508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,12,2,128,1,float16,float16,0,0.012191999703645706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,12,2,128,1,float16,fp8,0,0.012857599556446076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,12,2,128,1,fp8,fp8,0,0.012796799838542938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,12,4,128,1,float16,float16,0,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,12,4,128,1,float16,fp8,0,0.01343040019273758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,12,4,128,1,fp8,fp8,0,0.012742400169372559
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,12,1,128,1,fp8,fp8,0,0.013897599279880523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,12,12,128,1,float16,fp8,0,0.013230399787425995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,12,4,128,1,fp8,fp8,0,0.012383999675512314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,12,12,128,1,fp8,fp8,0,0.013419200479984284
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,12,4,128,1,float16,float16,0,0.01297439932823181
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,12,1,128,1,float16,fp8,0,0.012798400223255157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,12,1,128,1,fp8,fp8,0,0.012492799758911132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,12,2,128,1,float16,fp8,0,0.012302400171756744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,12,12,128,1,fp8,fp8,0,0.017788800597190856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,12,2,128,1,fp8,fp8,0,0.012294399738311767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,12,4,128,1,float16,float16,0,0.011708799749612808
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,12,4,128,1,float16,fp8,0,0.012681600451469422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,12,1,128,1,float16,fp8,0,0.013371199369430542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,12,12,128,1,float16,float16,0,0.012323199957609176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,12,12,128,1,float16,fp8,0,0.013196800649166108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,12,1,128,1,float16,float16,0,0.011988800019025803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,12,1,128,1,float16,fp8,0,0.012627199292182922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,12,1,128,1,fp8,fp8,0,0.01265919953584671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,12,2,128,1,float16,float16,0,0.01165120005607605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,12,2,128,1,float16,fp8,0,0.012377600371837615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,12,2,128,1,fp8,fp8,0,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,12,4,128,1,float16,float16,0,0.011748799681663513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,12,4,128,1,float16,fp8,0,0.012476799637079239
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,12,4,128,1,fp8,fp8,0,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,12,12,128,1,float16,float16,0,0.012057600170373916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,12,12,128,1,float16,fp8,0,0.012703999876976013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,12,1,128,1,float16,float16,0,0.012084800004959106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,12,12,128,1,fp8,fp8,0,0.012755200266838074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,12,1,128,1,float16,float16,0,0.01175680011510849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,12,1,128,1,float16,fp8,0,0.012479999661445617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,12,1,128,1,fp8,fp8,0,0.012454400211572647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,12,2,128,1,float16,float16,0,0.011552000045776367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,12,2,128,1,float16,fp8,0,0.012321600317955017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,12,2,128,1,fp8,fp8,0,0.012486399710178375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,12,4,128,1,float16,float16,0,0.011788800358772278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,12,4,128,1,float16,fp8,0,0.01250240057706833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,12,4,128,1,fp8,fp8,0,0.01249760016798973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,12,1,128,1,float16,float16,0,0.021273599565029146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,12,1,128,1,float16,fp8,0,0.022940799593925476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,12,1,128,1,fp8,fp8,0,0.022830399870872497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,12,2,128,1,float16,float16,0,0.02821280062198639
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,12,2,128,1,float16,fp8,0,0.031358399987220766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,12,1,128,1,float16,float16,0,0.011803200095891952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,12,2,128,1,float16,float16,0,0.011907199770212174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,12,4,128,1,float16,float16,0,0.043105599284172055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,12,4,128,1,float16,fp8,0,0.049089598655700686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,12,4,128,1,fp8,fp8,0,0.04915040135383606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,12,12,128,1,float16,float16,0,0.05584319829940796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,12,4,128,1,fp8,fp8,0,0.012531200051307678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,12,1,128,1,float16,float16,0,0.014707200229167938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,12,1,128,1,float16,fp8,0,0.015638400614261628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,12,1,128,1,fp8,fp8,0,0.015582400560379028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,12,2,128,1,float16,float16,0,0.018302400410175324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,12,2,128,1,float16,fp8,0,0.01996160000562668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,12,2,128,1,fp8,fp8,0,0.01977279931306839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,12,4,128,1,float16,float16,0,0.025784000754356384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,12,4,128,1,float16,fp8,0,0.028787198662757873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,12,4,128,1,fp8,fp8,0,0.028808000683784484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,12,12,128,1,float16,float16,0,0.03237920105457306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,12,12,128,1,float16,fp8,0,0.03721120059490204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,12,12,128,1,fp8,fp8,0,0.037201601266860965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,12,1,128,1,float16,float16,0,0.013284799456596375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,12,1,128,1,float16,fp8,0,0.014071999490261078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,12,1,128,1,fp8,fp8,0,0.014107200503349304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,12,2,128,1,float16,float16,0,0.013676799833774567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,12,2,128,1,float16,fp8,0,0.01438560038805008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,12,2,128,1,fp8,fp8,0,0.014416000247001648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,12,4,128,1,float16,float16,0,0.017347200214862822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,12,4,128,1,float16,fp8,0,0.018940800428390504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,12,4,128,1,fp8,fp8,0,0.01892320066690445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,12,12,128,1,float16,float16,0,0.02083519995212555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,12,12,128,1,float16,fp8,0,0.02287999987602234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,12,12,128,1,fp8,fp8,0,0.022864000499248506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,12,1,128,1,float16,float16,0,0.012583999335765839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,12,1,128,1,float16,fp8,0,0.013303999602794648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,12,1,128,1,fp8,fp8,0,0.013443200290203095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,12,12,128,1,float16,fp8,0,0.06500639915466308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,12,2,128,1,float16,float16,0,0.012763200700283051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,12,12,128,1,fp8,fp8,0,0.013123199343681335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,12,4,128,1,float16,float16,0,0.012625600397586822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,12,4,128,1,float16,fp8,0,0.013465599715709686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,12,4,128,1,fp8,fp8,0,0.013491199910640716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,12,12,128,1,float16,float16,0,0.016206400096416475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,12,12,128,1,float16,fp8,0,0.017387199401855468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,12,12,128,1,fp8,fp8,0,0.017560000717639922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,12,1,128,1,float16,float16,0,0.011897599697113037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,12,1,128,1,float16,fp8,0,0.012668800354003907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,12,1,128,1,fp8,fp8,0,0.012508800625801087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,12,2,128,1,float16,float16,0,0.011771199852228164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,12,2,128,1,float16,fp8,0,0.012476799637079239
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,12,2,128,1,fp8,fp8,0,0.012863999605178833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,12,4,128,1,float16,float16,0,0.011884800344705581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,12,4,128,1,float16,fp8,0,0.013035200536251068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,12,4,128,1,fp8,fp8,0,0.012628799676895142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,12,12,128,1,float16,float16,0,0.012601600587368011
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,12,12,128,1,float16,fp8,0,0.013334399461746216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,12,12,128,1,fp8,fp8,0,0.013465599715709686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,12,1,128,1,float16,float16,0,0.011713600158691407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,12,1,128,1,float16,fp8,0,0.012600000202655792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,12,1,128,1,fp8,fp8,0,0.012439999729394913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,12,2,128,1,float16,float16,0,0.011812800168991089
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,12,2,128,1,fp8,fp8,0,0.012479999661445617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,12,4,128,1,float16,float16,0,0.011795199662446975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,12,12,128,1,float16,float16,0,0.012359999865293504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,12,4,128,1,fp8,fp8,0,0.012907199561595917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,12,12,128,1,float16,float16,0,0.012345600128173827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,12,12,128,1,float16,fp8,0,0.01305440068244934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,12,12,128,1,fp8,fp8,0,0.013068799674510957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,12,1,128,1,float16,float16,0,0.011401599645614624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,12,1,128,1,float16,fp8,0,0.012334399670362473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,12,1,128,1,fp8,fp8,0,0.012582400441169738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,12,1,128,1,float16,float16,0,0.019083200395107268
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,12,2,128,1,float16,float16,0,0.011559999734163284
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,12,12,128,1,fp8,fp8,0,0.06529120206832886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,12,2,128,1,fp8,fp8,0,0.012321600317955017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,12,4,128,1,float16,float16,0,0.011687999963760376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,12,4,128,1,float16,fp8,0,0.012404800206422806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,12,4,128,1,fp8,fp8,0,0.012374400347471236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,12,12,128,1,float16,fp8,0,0.012895999848842621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,12,12,128,1,fp8,fp8,0,0.012876799702644348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,12,1,128,1,float16,float16,0,0.01157120019197464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,12,1,128,1,float16,fp8,0,0.012307199835777282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,12,1,128,1,fp8,fp8,0,0.01231039986014366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,12,2,128,1,float16,float16,0,0.011566399782896041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,12,2,128,1,float16,fp8,0,0.012257599830627441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,12,2,128,1,fp8,fp8,0,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,12,4,128,1,float16,float16,0,0.011433599889278412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,12,4,128,1,float16,fp8,0,0.012244799733161926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,12,4,128,1,fp8,fp8,0,0.012249600142240524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,12,12,128,1,float16,float16,0,0.01189119964838028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,12,12,128,1,float16,fp8,0,0.012611199915409089
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,12,12,128,1,fp8,fp8,0,0.012654399871826172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,12,1,128,1,float16,float16,0,0.01157120019197464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,12,2,128,1,float16,fp8,0,0.012488000094890594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,12,1,128,1,float16,fp8,0,0.012252800166606903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,12,1,128,1,fp8,fp8,0,0.012223999947309494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,12,2,128,1,float16,float16,0,0.011503999680280685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,12,2,128,1,float16,fp8,0,0.012057600170373916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,12,2,128,1,fp8,fp8,0,0.012059199810028075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,12,2,128,1,fp8,fp8,0,0.031414398550987245
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,12,4,128,1,float16,float16,0,0.011313600093126297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,12,4,128,1,float16,fp8,0,0.01231039986014366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,12,2,128,1,float16,fp8,0,0.013502399623394012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,12,12,128,1,float16,float16,0,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,12,4,128,1,float16,fp8,0,0.012577599287033081
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,12,4,128,1,fp8,fp8,0,0.012092799693346024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,12,2,128,1,float16,fp8,0,0.012399999797344208
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,8,1,128,1,float16,fp8,0,6.352664184570313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,8,1,128,1,fp8,fp8,0,6.42016830444336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,8,1,128,1,float16,float16,0,7.574761962890625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,12,2,128,1,fp8,fp8,0,0.013566400110721587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,8,2,128,1,float16,fp8,0,6.302004623413086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,8,2,128,1,float16,float16,0,7.4850914001464846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,8,2,128,1,fp8,fp8,0,6.380503845214844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,8,8,128,1,float16,float16,0,3.8394191741943358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,8,4,128,1,float16,float16,0,7.507657623291015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,8,4,128,1,float16,fp8,0,6.449880218505859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,8,8,128,1,fp8,fp8,0,3.258137512207031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,8,4,128,1,fp8,fp8,0,6.54083023071289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,8,1,128,1,float16,fp8,0,3.2171630859375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,8,1,128,1,float16,float16,0,3.713679885864258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,8,1,128,1,fp8,fp8,0,3.3436206817626952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,8,8,128,1,float16,fp8,0,3.215039825439453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,8,2,128,1,float16,fp8,0,3.193601608276367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,8,2,128,1,fp8,fp8,0,3.1835760116577148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,8,2,128,1,float16,float16,0,3.7695632934570313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,8,8,128,1,float16,fp8,0,1.6369663238525392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,8,4,128,1,float16,fp8,0,3.2308879852294923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,8,4,128,1,float16,float16,0,3.8808513641357423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,8,8,128,1,fp8,fp8,0,1.6620271682739258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,8,1,128,1,float16,float16,0,1.7205232620239257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,8,1,128,1,fp8,fp8,0,1.6018224716186524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,8,1,128,1,float16,fp8,0,1.6985952377319335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,8,2,128,1,float16,float16,0,1.8095903396606445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,8,2,128,1,float16,fp8,0,1.7464799880981445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,8,2,128,1,fp8,fp8,0,1.596247959136963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,8,8,128,1,float16,float16,0,1.0088624000549316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,8,4,128,1,fp8,fp8,0,1.6388015747070312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,8,4,128,1,float16,float16,0,1.9001888275146483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,8,8,128,1,float16,float16,0,1.8120351791381837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,8,8,128,1,fp8,fp8,0,0.8432111740112305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,8,1,128,1,float16,fp8,0,0.8003392219543457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,8,1,128,1,float16,float16,0,0.9739168167114258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,8,4,128,1,fp8,fp8,0,3.340340805053711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,8,1,128,1,fp8,fp8,0,0.8850239753723145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,8,2,128,1,float16,fp8,0,0.8029071807861328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,8,2,128,1,fp8,fp8,0,0.7981872081756591
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,8,4,128,1,float16,fp8,0,0.8133919715881348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,8,4,128,1,fp8,fp8,0,0.8504079818725586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,8,4,128,1,float16,fp8,0,1.6054464340209962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,8,8,128,1,float16,fp8,0,0.8560303688049317
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,8,2,128,1,float16,float16,0,0.8629664421081543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,8,1,128,1,float16,fp8,0,3.6371646881103517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,8,1,128,1,float16,float16,0,4.055329513549805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,8,1,128,1,fp8,fp8,0,3.6408336639404295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,8,4,128,1,float16,float16,0,0.9405839920043946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,8,2,128,1,float16,float16,0,4.160704040527344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,8,2,128,1,float16,fp8,0,3.609075164794922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,8,2,128,1,fp8,fp8,0,3.6473552703857424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,8,8,128,1,float16,fp8,0,1.8588544845581054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,8,8,128,1,float16,float16,0,2.1733728408813477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,8,8,128,1,fp8,fp8,0,2.0350368499755858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,8,4,128,1,float16,float16,0,4.201931381225586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,8,4,128,1,float16,fp8,0,3.6975807189941405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,8,1,128,1,float16,fp8,0,1.851036834716797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,8,4,128,1,fp8,fp8,0,3.8044670104980467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,8,1,128,1,fp8,fp8,0,1.8196271896362304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,8,2,128,1,float16,float16,0,2.144313621520996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,8,2,128,1,float16,fp8,0,1.8743551254272461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,8,8,128,1,float16,float16,0,1.0740063667297364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,8,2,128,1,fp8,fp8,0,1.9917072296142577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,8,8,128,1,float16,fp8,0,1.0750096321105957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,8,4,128,1,float16,fp8,0,1.8625104904174805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,8,4,128,1,fp8,fp8,0,1.8616767883300782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,8,8,128,1,fp8,fp8,0,0.9542688369750977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,8,1,128,1,float16,float16,0,1.014027214050293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,8,1,128,1,float16,fp8,0,0.9381440162658692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,8,1,128,1,fp8,fp8,0,0.9647215843200684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,8,2,128,1,float16,float16,0,0.992465591430664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,8,2,128,1,float16,fp8,0,0.9564319610595703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,8,2,128,1,fp8,fp8,0,0.9073519706726074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,8,1,128,1,float16,float16,0,1.9894912719726563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,8,4,128,1,float16,float16,0,1.1012080192565918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,8,4,128,1,float16,fp8,0,0.9301919937133789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,8,4,128,1,fp8,fp8,0,0.9403183937072754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,8,1,128,1,float16,float16,0,0.4870880126953125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,8,1,128,1,fp8,fp8,0,0.4562431812286377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,8,2,128,1,float16,float16,0,0.4892576217651367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,8,2,128,1,float16,fp8,0,0.45946078300476073
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,8,2,128,1,fp8,fp8,0,0.4564640045166016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,8,4,128,1,float16,float16,0,0.5175824165344238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,8,4,128,1,float16,fp8,0,0.46644001007080077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,8,4,128,1,fp8,fp8,0,0.4638656139373779
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,8,4,128,1,float16,float16,0,2.0484752655029297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,8,8,128,1,float16,fp8,0,0.530027198791504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,8,8,128,1,fp8,fp8,0,0.5008111953735351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,8,1,128,1,float16,fp8,0,2.533580780029297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,8,1,128,1,float16,fp8,0,0.544596815109253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,8,1,128,1,float16,float16,0,2.941948890686035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,8,1,128,1,fp8,fp8,0,2.5305871963500977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,8,2,128,1,float16,fp8,0,2.534436798095703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,8,2,128,1,float16,float16,0,2.7664079666137695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,8,2,128,1,fp8,fp8,0,2.554209518432617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,8,4,128,1,fp8,fp8,0,2.587086486816406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,8,4,128,1,float16,fp8,0,2.714567947387695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,8,8,128,1,float16,float16,0,0.5749231815338135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,8,8,128,1,float16,float16,0,1.4786368370056153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,8,8,128,1,float16,fp8,0,1.5096063613891602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,8,8,128,1,fp8,fp8,0,1.3543519973754883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,8,1,128,1,float16,float16,0,1.4022192001342773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,8,1,128,1,fp8,fp8,0,1.2780256271362305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,8,2,128,1,float16,fp8,0,1.3286687850952148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,8,2,128,1,fp8,fp8,0,1.2707455635070801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,8,4,128,1,float16,float16,0,1.4179120063781738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,8,4,128,1,fp8,fp8,0,1.2966912269592286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,8,4,128,1,float16,fp8,0,1.438081645965576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,8,8,128,1,float16,float16,0,0.7794303894042969
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,8,8,128,1,float16,fp8,0,0.6888224124908447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,8,8,128,1,fp8,fp8,0,0.6904655933380127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,8,1,128,1,float16,float16,0,0.6935247898101806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,8,4,128,1,float16,float16,0,3.0264768600463867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,8,1,128,1,fp8,fp8,0,0.6906015872955322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,8,2,128,1,float16,float16,0,0.694811201095581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,8,2,128,1,float16,fp8,0,0.668230390548706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,8,2,128,1,fp8,fp8,0,0.7001440048217773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,8,4,128,1,float16,float16,0,0.7196176052093506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,8,8,128,1,float16,float16,0,0.41945438385009765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,8,8,128,1,float16,fp8,0,0.39198400974273684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,8,1,128,1,float16,fp8,0,1.2931551933288574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,8,8,128,1,fp8,fp8,0,0.35814080238342283
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,8,1,128,1,float16,float16,0,0.3579200029373169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,8,1,128,1,float16,fp8,0,0.3598495960235596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,8,1,128,1,fp8,fp8,0,0.3314624071121216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,8,2,128,1,float16,float16,0,0.3577375888824463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,8,2,128,1,float16,fp8,0,0.3456592082977295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,8,2,128,1,float16,float16,0,1.4797712326049806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,8,4,128,1,float16,fp8,0,0.33647680282592773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,8,4,128,1,fp8,fp8,0,0.35274879932403563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,8,1,128,1,float16,fp8,0,0.669265604019165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,8,4,128,1,float16,fp8,0,0.6878159999847412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,8,4,128,1,fp8,fp8,0,0.7198368072509765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,8,1,128,1,float16,fp8,0,3.3048961639404295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,8,1,128,1,fp8,fp8,0,3.314311981201172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,8,2,128,1,fp8,fp8,0,0.3405087947845459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,8,1,128,1,float16,float16,0,3.830680084228516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,8,2,128,1,float16,float16,0,3.7634368896484376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,8,2,128,1,float16,fp8,0,3.309774398803711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,8,2,128,1,fp8,fp8,0,3.2976993560791015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,8,4,128,1,float16,fp8,0,3.4067073822021485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,8,8,128,1,float16,fp8,0,1.8523120880126953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,8,4,128,1,float16,float16,0,4.065867233276367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,8,8,128,1,fp8,fp8,0,1.742910385131836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,8,1,128,1,float16,float16,0,1.7643024444580078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,8,1,128,1,float16,fp8,0,1.6618944168090821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,8,1,128,1,fp8,fp8,0,1.6895551681518555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,8,2,128,1,float16,fp8,0,1.6658464431762696
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,8,2,128,1,float16,float16,0,1.986404800415039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,8,2,128,1,fp8,fp8,0,1.7347776412963867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,8,8,128,1,float16,fp8,0,0.8949392318725586
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,8,8,128,1,float16,float16,0,1.0604175567626952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,8,4,128,1,float16,fp8,0,1.68701114654541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,8,4,128,1,float16,float16,0,1.9061248779296875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,8,4,128,1,float16,float16,0,0.37257919311523435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,8,8,128,1,fp8,fp8,0,0.8789888381958008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,8,1,128,1,float16,float16,0,0.9672400474548339
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,8,1,128,1,fp8,fp8,0,0.9336544036865234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,8,2,128,1,float16,float16,0,0.8876432418823242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,8,2,128,1,float16,fp8,0,0.8313568115234375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,8,2,128,1,fp8,fp8,0,0.9350895881652832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,8,4,128,1,float16,float16,0,0.9361503601074219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,8,4,128,1,float16,fp8,0,0.8425616264343262
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,8,8,128,1,float16,float16,0,2.045035171508789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,8,8,128,1,float16,float16,0,0.49591679573059083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,8,4,128,1,fp8,fp8,0,0.8437888145446777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,8,8,128,1,float16,fp8,0,0.4681488037109375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,8,1,128,1,float16,float16,0,0.44220638275146484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,8,1,128,1,float16,fp8,0,0.43990240097045896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,8,1,128,1,fp8,fp8,0,0.41985602378845216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,8,2,128,1,float16,float16,0,0.45943517684936525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,8,2,128,1,float16,fp8,0,0.4228672027587891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,8,4,128,1,fp8,fp8,0,3.378123092651367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,8,2,128,1,fp8,fp8,0,0.4288640022277832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,8,4,128,1,float16,float16,0,0.4606800079345703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,8,4,128,1,float16,fp8,0,0.443990421295166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,8,4,128,1,fp8,fp8,0,1.6914384841918946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,8,8,128,1,float16,fp8,0,0.22966880798339845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,8,8,128,1,fp8,fp8,0,0.2257920026779175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,8,1,128,1,float16,float16,0,0.22909920215606688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,8,1,128,1,float16,fp8,0,0.2140064001083374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,8,1,128,1,fp8,fp8,0,0.2146224021911621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,8,2,128,1,float16,float16,0,0.2294640064239502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,8,2,128,1,float16,fp8,0,0.2165168046951294
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,8,2,128,1,fp8,fp8,0,0.21412160396575927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,8,4,128,1,float16,float16,0,0.23508639335632325
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,8,4,128,1,fp8,fp8,0,0.21752960681915284
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,8,8,128,1,fp8,fp8,0,0.44154558181762693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,8,1,128,1,float16,fp8,0,1.9185056686401367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,8,1,128,1,float16,float16,0,2.1205728530883787
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,8,8,128,1,float16,float16,0,0.24765279293060302
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,8,1,128,1,float16,fp8,0,0.8473456382751465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,8,1,128,1,fp8,fp8,0,1.9069072723388671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,8,2,128,1,float16,float16,0,2.0902624130249023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,8,2,128,1,float16,fp8,0,2.0765312194824217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,8,2,128,1,fp8,fp8,0,1.893110466003418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,8,4,128,1,float16,float16,0,2.0995407104492188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,8,8,128,1,float16,float16,0,1.140073585510254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,8,8,128,1,float16,fp8,0,1.0207280158996581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,8,4,128,1,fp8,fp8,0,2.1711824417114256
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,8,4,128,1,fp8,fp8,0,0.43003039360046386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,8,1,128,1,float16,float16,0,1.0260704040527344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,8,8,128,1,fp8,fp8,0,1.1290575981140136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,8,1,128,1,float16,fp8,0,0.9651424407958984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,8,4,128,1,float16,fp8,0,0.21637918949127197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,8,1,128,1,fp8,fp8,0,1.0167599678039552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,8,2,128,1,float16,float16,0,1.019260787963867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,8,2,128,1,float16,fp8,0,0.9662400245666504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,8,2,128,1,fp8,fp8,0,1.054368019104004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,8,4,128,1,float16,float16,0,1.0715120315551758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,8,4,128,1,float16,fp8,0,0.9794416427612305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,8,8,128,1,float16,fp8,0,0.5157440185546875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,8,8,128,1,fp8,fp8,0,0.516809606552124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,8,1,128,1,float16,float16,0,0.5327951908111572
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,8,1,128,1,float16,fp8,0,0.5238080024719238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,8,1,128,1,fp8,fp8,0,0.49113922119140624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,8,2,128,1,float16,float16,0,0.5125760078430176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,8,2,128,1,float16,fp8,0,0.48663997650146484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,8,2,128,1,fp8,fp8,0,0.5157408237457275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,8,4,128,1,float16,fp8,0,1.9533775329589844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,8,4,128,1,float16,fp8,0,0.49722561836242674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,8,8,128,1,float16,float16,0,0.3081952095031738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,8,4,128,1,fp8,fp8,0,0.49980959892272947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,8,8,128,1,fp8,fp8,0,0.27028961181640626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,8,1,128,1,float16,float16,0,0.26560320854187014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,8,1,128,1,float16,fp8,0,0.24736640453338624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,8,1,128,1,fp8,fp8,0,0.2493824005126953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,8,2,128,1,float16,float16,0,0.2644704103469849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,8,2,128,1,fp8,fp8,0,0.24801280498504638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,8,4,128,1,float16,float16,0,0.27715840339660647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,8,4,128,1,float16,fp8,0,0.2547856092453003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,8,4,128,1,fp8,fp8,0,0.2519808053970337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,8,8,128,1,float16,fp8,0,0.14415680170059203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,8,8,128,1,fp8,fp8,0,0.1428015947341919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,8,1,128,1,float16,float16,0,0.14455679655075074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,8,1,128,1,float16,fp8,0,0.1356271982192993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,8,1,128,1,fp8,fp8,0,0.13661919832229613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,8,2,128,1,float16,float16,0,0.14401439428329468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,8,2,128,1,float16,fp8,0,0.13671679496765138
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,8,2,128,1,fp8,fp8,0,0.13555359840393066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,8,4,128,1,float16,float16,0,0.15224159955978395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,8,4,128,1,float16,fp8,0,0.1363103985786438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,8,4,128,1,fp8,fp8,0,0.13715519905090331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,8,4,128,1,float16,float16,0,0.5444975852966308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,8,8,128,1,float16,fp8,0,0.2753216028213501
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,8,2,128,1,float16,fp8,0,0.2481775999069214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,8,1,128,1,float16,fp8,0,1.8117216110229493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,8,1,128,1,float16,float16,0,1.9468687057495118
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,8,8,128,1,float16,float16,0,0.16875200271606444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,8,1,128,1,fp8,fp8,0,1.8131584167480468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,8,2,128,1,float16,float16,0,1.987272071838379
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,8,8,128,1,float16,float16,0,0.5880879878997802
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,8,2,128,1,float16,fp8,0,1.8083440780639648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,8,2,128,1,fp8,fp8,0,1.8099088668823242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,8,4,128,1,float16,float16,0,2.1194944381713867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,8,4,128,1,float16,fp8,0,1.8597583770751953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,8,8,128,1,float16,float16,0,1.2006943702697754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,8,8,128,1,float16,fp8,0,0.9800848007202149
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,8,1,128,1,float16,float16,0,0.9509712219238281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,8,1,128,1,fp8,fp8,0,0.9124431610107422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,8,2,128,1,float16,float16,0,1.0589200019836427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,8,2,128,1,float16,fp8,0,0.9131199836730957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,8,2,128,1,fp8,fp8,0,0.9124496459960938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,8,4,128,1,float16,float16,0,0.9956512451171875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,8,8,128,1,float16,float16,0,0.5377520084381103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,8,4,128,1,fp8,fp8,0,0.9379551887512207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,8,4,128,1,float16,fp8,0,1.051022434234619
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,8,8,128,1,float16,fp8,0,0.49634881019592286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,8,8,128,1,fp8,fp8,0,0.5029712200164795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,8,4,128,1,fp8,fp8,0,0.9736751556396485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,8,1,128,1,float16,float16,0,0.4850768089294434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,8,1,128,1,fp8,fp8,0,0.4676447868347168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,8,2,128,1,float16,float16,0,0.48871359825134275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,8,2,128,1,float16,fp8,0,0.45943679809570315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,8,2,128,1,fp8,fp8,0,0.46265277862548826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,8,4,128,1,float16,float16,0,0.5033376216888428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,8,4,128,1,float16,fp8,0,0.4764688014984131
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,8,4,128,1,fp8,fp8,0,0.4726255893707275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,8,8,128,1,fp8,fp8,0,0.9740127563476563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,8,8,128,1,float16,fp8,0,0.25645599365234373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,8,8,128,1,fp8,fp8,0,0.25552639961242674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,8,1,128,1,float16,float16,0,0.24508159160614013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,8,1,128,1,float16,fp8,0,0.23769919872283934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,8,1,128,1,float16,fp8,0,1.0097056388854981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,8,1,128,1,fp8,fp8,0,0.2382863998413086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,8,2,128,1,float16,float16,0,0.24893438816070557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,8,2,128,1,float16,fp8,0,0.23612160682678224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,8,2,128,1,fp8,fp8,0,0.2381743907928467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,8,4,128,1,float16,float16,0,0.2586080074310303
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,8,4,128,1,float16,fp8,0,0.2436959981918335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,8,8,128,1,float16,fp8,0,0.13367840051651
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,8,8,128,1,fp8,fp8,0,0.13358559608459472
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,8,1,128,1,float16,float16,0,0.12630079984664916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,8,1,128,1,float16,fp8,0,0.12028160095214843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,8,2,128,1,float16,float16,0,0.12984800338745117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,8,2,128,1,float16,fp8,0,0.12086080312728882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,8,2,128,1,fp8,fp8,0,0.12088479995727539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,8,4,128,1,float16,float16,0,0.13439040184020995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,8,1,128,1,float16,fp8,0,0.4938352108001709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,8,4,128,1,fp8,fp8,0,0.123089599609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,8,8,128,1,float16,float16,0,0.07540000081062317
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,8,8,128,1,float16,fp8,0,0.06863840222358704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,8,8,128,1,fp8,fp8,0,0.06839200258255004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,8,8,128,1,float16,float16,0,0.27738399505615235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,8,1,128,1,float16,float16,0,0.06635199785232544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,8,1,128,1,float16,fp8,0,0.06452000141143799
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,8,1,128,1,fp8,fp8,0,0.06448320150375367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,8,2,128,1,float16,float16,0,0.0669103980064392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,8,2,128,1,fp8,fp8,0,0.06415039896965027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,8,4,128,1,float16,float16,0,0.06857920289039612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,8,4,128,1,float16,fp8,0,0.06633440256118775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,8,8,128,1,float16,float16,0,0.14619359970092774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,8,4,128,1,fp8,fp8,0,0.2422879934310913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,8,1,128,1,fp8,fp8,0,0.12074559926986694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,8,1,128,1,float16,float16,0,1.1353887557983398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,8,1,128,1,fp8,fp8,0,1.0822832107543945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,8,4,128,1,float16,fp8,0,0.12390400171279907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,8,2,128,1,float16,float16,0,1.1734944343566895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,8,2,128,1,float16,fp8,0,1.0812687873840332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,8,2,128,1,fp8,fp8,0,1.0819616317749023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,8,2,128,1,float16,fp8,0,0.06424319744110107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,8,4,128,1,fp8,fp8,0,0.06638560295104981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,8,8,128,1,float16,float16,0,0.6593088150024414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,8,4,128,1,float16,float16,0,1.2933775901794433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,8,4,128,1,float16,fp8,0,1.1208368301391602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,8,4,128,1,fp8,fp8,0,1.1181407928466798
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,8,8,128,1,float16,fp8,0,0.5980288028717041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,8,8,128,1,fp8,fp8,0,0.5971151828765869
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,8,1,128,1,float16,float16,0,0.6189199924468994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,8,1,128,1,float16,fp8,0,0.5457039833068847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,8,1,128,1,float16,fp8,0,1.0772432327270507
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,8,2,128,1,float16,float16,0,0.5672607898712159
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,8,2,128,1,float16,fp8,0,0.5483712196350098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,8,2,128,1,fp8,fp8,0,0.5482944011688232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,8,4,128,1,float16,float16,0,0.6373824119567871
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,8,4,128,1,float16,fp8,0,0.563374376296997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,8,4,128,1,fp8,fp8,0,0.5634943962097168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,8,8,128,1,fp8,fp8,0,0.30479679107666013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,8,1,128,1,float16,float16,0,0.310099196434021
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,8,1,128,1,fp8,fp8,0,0.27931358814239504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,8,2,128,1,float16,float16,0,0.29397599697113036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,8,2,128,1,float16,fp8,0,0.30385921001434324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,8,4,128,1,float16,float16,0,0.30650880336761477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,8,4,128,1,float16,fp8,0,0.30617918968200686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,8,4,128,1,fp8,fp8,0,0.28765439987182617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,8,8,128,1,float16,float16,0,0.18449440002441406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,8,8,128,1,float16,fp8,0,0.1574399948120117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,8,8,128,1,fp8,fp8,0,0.16131999492645263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,8,1,128,1,float16,float16,0,0.15023039579391478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,8,1,128,1,float16,fp8,0,0.14819200038909913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,8,1,128,1,fp8,fp8,0,0.14453599452972413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,8,2,128,1,float16,float16,0,0.15555839538574218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,8,2,128,1,float16,fp8,0,0.14438079595565795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,8,2,128,1,fp8,fp8,0,0.14655040502548217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,8,4,128,1,float16,float16,0,0.15962079763412476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,8,4,128,1,float16,fp8,0,0.15182240009307862
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,8,4,128,1,fp8,fp8,0,0.14997919797897338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,8,8,128,1,float16,float16,0,0.09891359806060791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,8,8,128,1,float16,fp8,0,0.0855135977268219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,8,1,128,1,float16,float16,0,0.0813647985458374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,8,8,128,1,float16,float16,0,0.33741919994354247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,8,1,128,1,float16,fp8,0,0.07835360169410706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,8,1,128,1,fp8,fp8,0,0.07840480208396912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,8,2,128,1,float16,float16,0,0.08211359977722169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,8,1,128,1,float16,fp8,0,0.27999680042266845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,8,2,128,1,float16,fp8,0,0.07815679907798767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,8,2,128,1,fp8,fp8,0,0.07831360101699829
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,8,4,128,1,fp8,fp8,0,1.8622831344604491
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,8,4,128,1,float16,float16,0,0.08813920021057128
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,8,4,128,1,fp8,fp8,0,0.07840800285339355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,8,8,128,1,float16,fp8,0,0.05059679746627808
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,8,8,128,1,fp8,fp8,0,0.05127840042114258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,8,1,128,1,float16,float16,0,0.049744001030921935
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,8,1,128,1,float16,fp8,0,0.04905920028686524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,8,1,128,1,fp8,fp8,0,0.04993920028209686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,8,2,128,1,float16,fp8,0,0.04986400008201599
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,8,2,128,1,fp8,fp8,0,0.04966880083084106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,8,4,128,1,float16,float16,0,0.05081440210342407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,8,4,128,1,float16,fp8,0,0.04979679882526398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,8,4,128,1,fp8,fp8,0,0.04961600005626678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,8,1,128,1,fp8,fp8,0,0.5481503963470459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,8,1,128,1,float16,float16,0,1.0871968269348145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,8,8,128,1,fp8,fp8,0,0.08628960251808167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,8,8,128,1,float16,fp8,0,0.3050544023513794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,8,1,128,1,float16,fp8,0,1.0605152130126954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,8,4,128,1,float16,fp8,0,0.07868000268936157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,8,1,128,1,fp8,fp8,0,1.0602160453796388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,8,2,128,1,fp8,fp8,0,0.2797408103942871
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,8,2,128,1,float16,float16,0,1.1260895729064941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,8,2,128,1,float16,float16,0,0.05005120038986206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,8,2,128,1,fp8,fp8,0,1.061190414428711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,8,8,128,1,float16,float16,0,0.663759994506836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,8,4,128,1,float16,float16,0,1.1839247703552247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,8,4,128,1,float16,fp8,0,1.1050751686096192
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,8,4,128,1,fp8,fp8,0,1.10763521194458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,8,8,128,1,float16,fp8,0,0.6029679775238037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,8,8,128,1,fp8,fp8,0,0.6009088039398194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,8,1,128,1,float16,float16,0,0.5716320037841797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,8,1,128,1,fp8,fp8,0,0.539079999923706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,8,2,128,1,float16,float16,0,0.5573423862457275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,8,2,128,1,float16,fp8,0,0.5356304168701171
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,8,2,128,1,fp8,fp8,0,0.5620207786560059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,8,4,128,1,float16,float16,0,0.6031328201293945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,8,4,128,1,float16,fp8,0,0.5608911991119385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,8,8,128,1,float16,fp8,0,0.3172175884246826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,8,8,128,1,fp8,fp8,0,0.3077935934066772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,8,8,128,1,float16,float16,0,0.054451197385787964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,8,1,128,1,float16,float16,0,0.2896176099777222
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,8,1,128,1,float16,fp8,0,0.27531359195709226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,8,1,128,1,fp8,fp8,0,0.27739999294281004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,8,2,128,1,float16,float16,0,0.2917504072189331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,8,2,128,1,float16,fp8,0,0.27556960582733153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,8,2,128,1,fp8,fp8,0,0.2777616024017334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,8,4,128,1,float16,float16,0,0.30914719104766847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,8,4,128,1,float16,fp8,0,0.286627197265625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,8,8,128,1,float16,fp8,0,0.16117919683456422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,8,8,128,1,fp8,fp8,0,0.1604367971420288
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,8,1,128,1,float16,fp8,0,0.5380239963531495
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,8,1,128,1,float16,float16,0,0.14676640033721924
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,8,1,128,1,float16,fp8,0,0.14528319835662842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,8,1,128,1,fp8,fp8,0,0.14478399753570556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,8,2,128,1,float16,float16,0,0.15045759677886963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,8,2,128,1,fp8,fp8,0,0.14482719898223878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,8,4,128,1,float16,float16,0,0.15975359678268433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,8,8,128,1,float16,float16,0,0.34132320880889894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,8,4,128,1,fp8,fp8,0,0.557916784286499
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,8,4,128,1,fp8,fp8,0,0.15071680545806884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,8,8,128,1,float16,float16,0,0.09376000165939331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,8,8,128,1,fp8,fp8,0,0.08643360137939453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,8,1,128,1,float16,fp8,0,0.075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,8,2,128,1,float16,fp8,0,1.0530415534973145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,8,2,128,1,float16,float16,0,0.07981600165367127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,8,2,128,1,float16,fp8,0,0.0753600001335144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,8,2,128,1,fp8,fp8,0,0.07570239901542664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,8,8,128,1,float16,float16,0,0.1788383960723877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,8,4,128,1,float16,float16,0,0.0851631999015808
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,8,4,128,1,float16,fp8,0,0.07863360047340393
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,8,8,128,1,float16,float16,0,0.05169600248336792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,8,8,128,1,float16,fp8,0,0.045763200521469115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,8,8,128,1,fp8,fp8,0,0.04540640115737915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,8,1,128,1,float16,float16,0,0.04237279891967773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,8,1,128,1,float16,fp8,0,0.042296001315116884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,8,1,128,1,fp8,fp8,0,0.042752000689506534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,8,2,128,1,float16,fp8,0,0.145033597946167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,8,2,128,1,float16,float16,0,0.04282079935073853
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,8,2,128,1,float16,fp8,0,0.04187679886817932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,8,2,128,1,fp8,fp8,0,0.04213440120220184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,8,4,128,1,float16,float16,0,0.043875199556350705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,8,4,128,1,float16,fp8,0,0.042612800002098085
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,8,4,128,1,fp8,fp8,0,0.042473599314689636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,8,8,128,1,float16,fp8,0,0.03694080114364624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,8,4,128,1,float16,fp8,0,0.1500048041343689
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,8,8,128,1,fp8,fp8,0,0.0371071994304657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,8,1,128,1,float16,float16,0,0.036452800035476685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,8,1,128,1,float16,float16,0,0.0773248016834259
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,8,1,128,1,fp8,fp8,0,0.036160001158714296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,8,2,128,1,float16,float16,0,0.036847999691963194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,8,2,128,1,float16,fp8,0,0.03628320097923279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,8,2,128,1,fp8,fp8,0,0.03627679944038391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,8,1,128,1,fp8,fp8,0,0.07487679719924926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,8,4,128,1,float16,fp8,0,0.03624320030212402
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,8,4,128,1,fp8,fp8,0,0.036315199732780454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,8,4,128,1,fp8,fp8,0,0.28797919750213624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,8,1,128,1,float16,float16,0,0.7511824131011963
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,8,1,128,1,float16,fp8,0,0.7363264083862304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,8,1,128,1,fp8,fp8,0,0.7388031959533692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,8,2,128,1,float16,float16,0,0.766476821899414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,8,2,128,1,float16,fp8,0,0.7363872051239013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,8,8,128,1,float16,float16,0,0.03883199989795685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,8,2,128,1,fp8,fp8,0,0.7389440059661865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,8,1,128,1,float16,fp8,0,0.03616800010204315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,8,4,128,1,float16,float16,0,0.814020824432373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,8,4,128,1,float16,fp8,0,0.7725520133972168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,8,4,128,1,float16,float16,0,0.03740319907665253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,8,8,128,1,float16,float16,0,0.46404638290405276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,8,4,128,1,fp8,fp8,0,0.7796527862548828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,8,8,128,1,float16,fp8,0,0.4272064208984375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,8,8,128,1,fp8,fp8,0,0.4264863967895508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,8,1,128,1,float16,float16,0,0.383950400352478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,8,1,128,1,fp8,fp8,0,0.3779504060745239
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,8,2,128,1,float16,float16,0,0.3924015998840332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,8,4,128,1,fp8,fp8,0,0.07721440196037292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,8,2,128,1,fp8,fp8,0,0.37702240943908694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,8,4,128,1,float16,float16,0,0.41590237617492676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,8,8,128,1,float16,float16,0,0.24130239486694335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,8,4,128,1,float16,fp8,0,0.39698081016540526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,8,8,128,1,float16,fp8,0,0.22086720466613768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,8,8,128,1,fp8,fp8,0,0.22118558883666992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,8,1,128,1,float16,float16,0,0.2002880096435547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,8,1,128,1,fp8,fp8,0,0.19519200325012206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,8,2,128,1,float16,float16,0,0.2046288013458252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,8,2,128,1,float16,fp8,0,0.1957136034965515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,8,2,128,1,fp8,fp8,0,0.19680160284042358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,8,4,128,1,float16,float16,0,0.21624479293823243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,8,8,128,1,float16,fp8,0,0.08677600026130676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,8,4,128,1,float16,fp8,0,0.20552480220794678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,8,4,128,1,fp8,fp8,0,0.20494720935821534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,8,8,128,1,float16,fp8,0,0.11845439672470093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,8,8,128,1,fp8,fp8,0,0.1180832028388977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,8,1,128,1,float16,float16,0,0.10746079683303833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,8,1,128,1,float16,fp8,0,0.10436960458755493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,8,1,128,1,fp8,fp8,0,0.104476797580719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,8,1,128,1,float16,fp8,0,0.3762847900390625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,8,2,128,1,float16,float16,0,0.10974080562591552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,8,2,128,1,float16,fp8,0,0.3772320032119751
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,8,2,128,1,fp8,fp8,0,0.10530879497528076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,8,4,128,1,float16,float16,0,0.11607520580291748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,8,4,128,1,fp8,fp8,0,0.10990400314331054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,8,8,128,1,float16,fp8,0,0.06420639753341675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,8,4,128,1,fp8,fp8,0,0.39635679721832273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,8,8,128,1,fp8,fp8,0,0.06433759927749634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,8,1,128,1,float16,float16,0,0.05791680216789245
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,8,1,128,1,float16,fp8,0,0.19547040462493898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,8,1,128,1,fp8,fp8,0,0.057555198669433594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,8,2,128,1,float16,float16,0,0.058894401788711546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,8,2,128,1,fp8,fp8,0,0.05705440044403076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,8,4,128,1,float16,float16,0,0.06476160287857055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,8,4,128,1,float16,fp8,0,0.05811840295791626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,8,4,128,1,fp8,fp8,0,0.058379197120666505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,8,8,128,1,float16,float16,0,0.0392304003238678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,8,8,128,1,float16,fp8,0,0.03570559918880463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,8,8,128,1,float16,float16,0,0.1282047986984253
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,8,8,128,1,fp8,fp8,0,0.03579199910163879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,8,1,128,1,float16,fp8,0,0.034694400429725644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,8,1,128,1,fp8,fp8,0,0.034667199850082396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,8,2,128,1,float16,float16,0,0.0346015989780426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,8,2,128,1,float16,fp8,0,0.03460800051689148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,8,2,128,1,fp8,fp8,0,0.03452799916267395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,8,2,128,1,float16,fp8,0,0.10515199899673462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,8,4,128,1,float16,float16,0,0.035648000240325925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,8,4,128,1,float16,fp8,0,0.0352400004863739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,8,4,128,1,fp8,fp8,0,0.034944000840187076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,8,8,128,1,float16,float16,0,0.03972159922122955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,8,8,128,1,float16,fp8,0,0.03995679914951324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,8,4,128,1,float16,fp8,0,0.11012480258941651
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,8,1,128,1,float16,fp8,0,0.03440479934215546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,8,1,128,1,fp8,fp8,0,0.03410719931125641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,8,2,128,1,float16,float16,0,0.03688960075378418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,8,2,128,1,float16,fp8,0,0.03721440136432648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,8,2,128,1,fp8,fp8,0,0.03735840022563934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,8,4,128,1,float16,float16,0,0.038494399189949034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,8,4,128,1,float16,fp8,0,0.039478400349617006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,8,4,128,1,fp8,fp8,0,0.0395583987236023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,8,2,128,1,float16,fp8,0,0.057532799243927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,8,1,128,1,float16,float16,0,0.7985968112945556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,8,1,128,1,float16,fp8,0,0.7923920154571533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,8,1,128,1,fp8,fp8,0,0.7950240135192871
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,8,1,128,1,float16,float16,0,0.03440479934215546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,8,2,128,1,float16,float16,0,0.8182095527648926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,8,2,128,1,float16,fp8,0,0.7937439918518067
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,8,2,128,1,fp8,fp8,0,0.7959167957305908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,8,8,128,1,fp8,fp8,0,0.03972960114479065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,8,4,128,1,float16,float16,0,0.8774304389953613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,8,8,128,1,float16,float16,0,0.0711359977722168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,8,4,128,1,float16,fp8,0,0.8459487915039062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,8,8,128,1,float16,float16,0,0.5163968086242676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,8,1,128,1,float16,fp8,0,0.05706560015678406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,8,8,128,1,float16,fp8,0,0.4752495765686035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,8,1,128,1,float16,float16,0,0.4104335784912109
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,8,8,128,1,fp8,fp8,0,0.47452797889709475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,8,1,128,1,float16,fp8,0,0.4069056034088135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,8,2,128,1,float16,fp8,0,0.40603837966918943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,8,2,128,1,fp8,fp8,0,0.40731520652770997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,8,4,128,1,float16,fp8,0,0.4317024230957031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,8,8,128,1,float16,float16,0,0.26782879829406736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,8,4,128,1,fp8,fp8,0,0.43001279830932615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,8,8,128,1,float16,fp8,0,0.24718079566955567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,8,8,128,1,fp8,fp8,0,0.24687519073486328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,8,1,128,1,float16,float16,0,0.2143455982208252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,8,1,128,1,float16,fp8,0,0.21210720539093017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,8,1,128,1,fp8,fp8,0,0.21192638874053954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,8,2,128,1,float16,float16,0,0.21888320446014403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,8,2,128,1,float16,fp8,0,0.2127376079559326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,8,2,128,1,fp8,fp8,0,0.21291038990020753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,8,4,128,1,float16,float16,0,0.23479039669036866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,8,4,128,1,float16,fp8,0,0.2248863935470581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,8,1,128,1,float16,float16,0,0.03358240127563476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,8,8,128,1,float16,fp8,0,0.13231359720230101
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,8,8,128,1,fp8,fp8,0,0.13275680541992188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,8,1,128,1,float16,float16,0,0.11525440216064453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,8,1,128,1,float16,fp8,0,0.11425280570983887
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,8,1,128,1,fp8,fp8,0,0.11442240476608276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,8,2,128,1,float16,float16,0,0.11914880275726318
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,8,1,128,1,fp8,fp8,0,0.40515837669372556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,8,2,128,1,fp8,fp8,0,0.11451200246810914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,8,2,128,1,float16,float16,0,0.41831040382385254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,8,4,128,1,float16,float16,0,0.12742719650268555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,8,4,128,1,float16,float16,0,0.44951682090759276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,8,4,128,1,float16,fp8,0,0.12041759490966797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,8,8,128,1,float16,float16,0,0.07937759757041932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,8,8,128,1,float16,fp8,0,0.07480000257492066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,8,8,128,1,fp8,fp8,0,0.07512800097465515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,8,1,128,1,float16,float16,0,0.06437600255012513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,8,1,128,1,float16,fp8,0,0.0631056010723114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,8,1,128,1,fp8,fp8,0,0.06320160031318664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,8,2,128,1,float16,float16,0,0.066593599319458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,8,2,128,1,float16,fp8,0,0.0637440025806427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,8,2,128,1,fp8,fp8,0,0.06364799737930298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,8,4,128,1,float16,float16,0,0.0719760000705719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,8,4,128,1,float16,fp8,0,0.06704480051994324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,8,4,128,1,fp8,fp8,0,0.0669376015663147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,8,8,128,1,float16,fp8,0,0.04211840033531189
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,8,8,128,1,fp8,fp8,0,0.04199039936065674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,8,1,128,1,float16,float16,0,0.039340800046920775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,8,1,128,1,float16,fp8,0,0.040031999349594116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,8,4,128,1,fp8,fp8,0,0.22443840503692628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,8,1,128,1,fp8,fp8,0,0.04017280042171478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,8,2,128,1,float16,float16,0,0.03985120058059692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,8,2,128,1,float16,fp8,0,0.04011200070381164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,8,4,128,1,fp8,fp8,0,0.8428671836853028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,8,4,128,1,float16,float16,0,0.041591998934745786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,8,4,128,1,fp8,fp8,0,0.040673598647117615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,8,4,128,1,float16,fp8,0,0.04045119881629944
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,8,8,128,1,float16,float16,0,0.029016000032424927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,8,2,128,1,float16,fp8,0,0.11483039855957031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,8,8,128,1,fp8,fp8,0,0.028604799509048463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,8,1,128,1,float16,float16,0,0.02685759961605072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,8,1,128,1,fp8,fp8,0,0.027628800272941588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,8,2,128,1,float16,float16,0,0.027164798974990845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,8,2,128,1,float16,fp8,0,0.02776319980621338
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,8,4,128,1,fp8,fp8,0,0.12032639980316162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,8,4,128,1,float16,float16,0,0.027798399329185486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,8,4,128,1,float16,fp8,0,0.028169599175453187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,8,4,128,1,fp8,fp8,0,0.028084799647331238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,8,8,128,1,float16,float16,0,0.02620159983634949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,8,8,128,1,float16,fp8,0,0.027385601401329042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,8,1,128,1,float16,float16,0,0.025352001190185547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,8,1,128,1,float16,fp8,0,0.026892799139022826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,8,1,128,1,fp8,fp8,0,0.026940798759460448
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,8,8,128,1,float16,float16,0,0.04837920069694519
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,8,2,128,1,float16,float16,0,0.025297600030899047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,8,2,128,1,float16,fp8,0,0.026683199405670165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,8,4,128,1,float16,float16,0,0.02553279995918274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,8,4,128,1,float16,fp8,0,0.02680639922618866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,8,4,128,1,fp8,fp8,0,0.026939201354980468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,8,8,128,1,float16,float16,0,0.14254720211029054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,8,2,128,1,fp8,fp8,0,0.04019359946250915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,8,1,128,1,float16,float16,0,0.49816322326660156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,8,1,128,1,float16,fp8,0,0.517955207824707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,8,8,128,1,float16,fp8,0,0.02863360047340393
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,8,1,128,1,float16,fp8,0,0.027721598744392395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,8,2,128,1,float16,float16,0,0.518620777130127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,8,2,128,1,float16,fp8,0,0.5183536052703858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,8,2,128,1,fp8,fp8,0,0.5191823959350585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,8,2,128,1,fp8,fp8,0,0.027811199426651
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,8,4,128,1,float16,float16,0,0.5862815856933594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,8,4,128,1,float16,fp8,0,0.5631455898284912
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,8,8,128,1,float16,float16,0,0.36608960628509524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,8,4,128,1,fp8,fp8,0,0.5666656017303466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,8,8,128,1,fp8,fp8,0,0.3396320104598999
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,8,1,128,1,float16,float16,0,0.25677759647369386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,8,8,128,1,fp8,fp8,0,0.027031999826431275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,8,1,128,1,float16,fp8,0,0.2672096014022827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,8,1,128,1,fp8,fp8,0,0.26646080017089846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,8,2,128,1,float16,float16,0,0.2682287931442261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,8,2,128,1,float16,fp8,0,0.26539039611816406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,8,2,128,1,fp8,fp8,0,0.26706719398498535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,8,4,128,1,float16,float16,0,0.3005808115005493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,8,8,128,1,float16,float16,0,0.1905519962310791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,8,4,128,1,fp8,fp8,0,0.2906208038330078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,8,8,128,1,float16,fp8,0,0.17721920013427733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,8,8,128,1,fp8,fp8,0,0.17743200063705444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,8,1,128,1,float16,float16,0,0.135972797870636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,8,1,128,1,float16,fp8,0,0.14066239595413207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,8,1,128,1,fp8,fp8,0,0.14096959829330444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,8,2,128,1,float16,float16,0,0.14203840494155884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,8,2,128,1,float16,fp8,0,0.14016640186309814
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,8,2,128,1,fp8,fp8,0,0.14081759452819825
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,8,4,128,1,float16,float16,0,0.15847840309143066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,8,4,128,1,float16,fp8,0,0.1530832052230835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,8,8,128,1,float16,float16,0,0.10280159711837769
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,8,8,128,1,float16,fp8,0,0.09635840058326721
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,8,8,128,1,fp8,fp8,0,0.09589759707450866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,8,1,128,1,float16,float16,0,0.07561600208282471
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,8,1,128,1,fp8,fp8,0,0.07707679867744446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,8,2,128,1,float16,float16,0,0.0785040020942688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,8,2,128,1,float16,fp8,0,0.07699840068817139
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,8,1,128,1,fp8,fp8,0,0.5181407928466797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,8,4,128,1,float16,float16,0,0.08666239976882935
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,8,4,128,1,float16,fp8,0,0.08384479880332947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,8,4,128,1,fp8,fp8,0,0.0839631974697113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,8,8,128,1,float16,float16,0,0.05927039980888367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,8,8,128,1,float16,fp8,0,0.054782402515411374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,8,8,128,1,fp8,fp8,0,0.05514559745788574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,8,1,128,1,float16,fp8,0,0.043136000633239746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,8,8,128,1,float16,fp8,0,0.3409487962722778
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,8,1,128,1,fp8,fp8,0,0.04337440133094787
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,8,2,128,1,float16,float16,0,0.045612800121307376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,8,2,128,1,float16,fp8,0,0.043540799617767335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,8,2,128,1,fp8,fp8,0,0.043558400869369504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,8,4,128,1,float16,float16,0,0.05060160160064697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,8,4,128,1,float16,fp8,0,0.04602720141410828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,8,4,128,1,fp8,fp8,0,0.046105599403381346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,8,8,128,1,float16,float16,0,0.03619360029697418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,8,8,128,1,float16,fp8,0,0.031518399715423584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,8,8,128,1,fp8,fp8,0,0.031692799925804135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,8,1,128,1,float16,float16,0,0.02720159888267517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,8,4,128,1,fp8,fp8,0,0.15186879634857178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,8,1,128,1,float16,fp8,0,0.028972798585891725
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,8,1,128,1,fp8,fp8,0,0.028721600770950317
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,8,2,128,1,float16,float16,0,0.02744159996509552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,8,2,128,1,float16,fp8,0,0.028542399406433105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,8,2,128,1,fp8,fp8,0,0.028702399134635924
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,8,4,128,1,float16,float16,0,0.028723201155662535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,8,2,128,1,fp8,fp8,0,0.07769119739532471
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,8,4,128,1,fp8,fp8,0,0.028999999165534973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,8,8,128,1,float16,float16,0,0.01995680034160614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,8,8,128,1,float16,fp8,0,0.020366400480270386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,8,8,128,1,fp8,fp8,0,0.02032800018787384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,8,1,128,1,float16,float16,0,0.017822399735450745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,8,1,128,1,float16,fp8,0,0.019438399374485014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,8,1,128,1,fp8,fp8,0,0.019254399836063384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,8,2,128,1,float16,float16,0,0.017884799838066102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,8,2,128,1,float16,fp8,0,0.019252799451351166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,8,1,128,1,float16,float16,0,0.042419201135635375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,8,4,128,1,float16,fp8,0,0.019486400485038757
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,8,4,128,1,fp8,fp8,0,0.019828799366950988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,8,8,128,1,float16,float16,0,0.018479999899864197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,8,8,128,1,float16,fp8,0,0.01966080069541931
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,8,8,128,1,fp8,fp8,0,0.019683200120925903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,8,1,128,1,float16,float16,0,0.017395199835300447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,8,1,128,1,float16,fp8,0,0.01887200027704239
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,8,1,128,1,fp8,fp8,0,0.018883199989795686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,8,2,128,1,float16,float16,0,0.01736160069704056
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,8,2,128,1,fp8,fp8,0,0.026625600457191468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,8,2,128,1,float16,fp8,0,0.018651199340820313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,8,4,128,1,float16,float16,0,0.017977599799633027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,8,4,128,1,float16,fp8,0,0.01923840045928955
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,8,4,128,1,fp8,fp8,0,0.019281600415706635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,8,8,128,1,float16,float16,0,0.017399999499320983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,8,8,128,1,float16,fp8,0,0.019252799451351166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,8,8,128,1,fp8,fp8,0,0.01918559968471527
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,8,1,128,1,float16,float16,0,0.016964800655841827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,8,1,128,1,float16,fp8,0,0.018596799671649934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,8,1,128,1,float16,fp8,0,0.07778080105781555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,8,1,128,1,fp8,fp8,0,0.018489600718021394
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,8,2,128,1,float16,float16,0,0.01690559983253479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,8,2,128,1,float16,fp8,0,0.018644799292087556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,8,4,128,1,float16,fp8,0,0.0289792001247406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,8,4,128,1,float16,float16,0,0.017371200025081635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,8,4,128,1,fp8,fp8,0,0.01894560009241104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,8,1,128,1,float16,float16,0,0.21238400936126708
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,8,1,128,1,float16,fp8,0,0.2246864080429077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,8,4,128,1,float16,float16,0,0.01870400011539459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,8,1,128,1,fp8,fp8,0,0.22434399127960206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,8,2,128,1,float16,float16,0,0.2234544038772583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,8,2,128,1,float16,fp8,0,0.22418560981750488
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,8,2,128,1,fp8,fp8,0,0.22471199035644532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,8,4,128,1,float16,float16,0,0.2564383983612061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,8,4,128,1,float16,fp8,0,0.2494704008102417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,8,4,128,1,fp8,fp8,0,0.24936480522155763
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,8,8,128,1,float16,float16,0,0.17015360593795775
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,8,8,128,1,float16,fp8,0,0.148526394367218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,8,8,128,1,fp8,fp8,0,0.14920320510864257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,8,4,128,1,float16,fp8,0,0.29066879749298097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,8,1,128,1,float16,fp8,0,0.11858079433441163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,8,1,128,1,fp8,fp8,0,0.11896799802780152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,8,2,128,1,fp8,fp8,0,0.018676799535751343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,8,2,128,1,float16,float16,0,0.11895999908447266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,8,2,128,1,float16,fp8,0,0.11886880397796631
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,8,2,128,1,fp8,fp8,0,0.11941759586334229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,8,4,128,1,float16,fp8,0,0.1318336009979248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,8,4,128,1,float16,float16,0,0.13551199436187744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,8,8,128,1,float16,float16,0,0.09329919815063477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,8,8,128,1,float16,fp8,0,0.0816431999206543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,8,8,128,1,fp8,fp8,0,0.0818560004234314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,8,1,128,1,float16,fp8,0,0.06613119840621948
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,8,1,128,1,fp8,fp8,0,0.06551520228385925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,8,2,128,1,float16,float16,0,0.06723359823226929
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,8,2,128,1,float16,fp8,0,0.06620320081710815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,8,2,128,1,fp8,fp8,0,0.06578720211982728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,8,4,128,1,float16,float16,0,0.07550399899482726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,8,4,128,1,float16,fp8,0,0.072188800573349
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,8,4,128,1,float16,fp8,0,0.01903519928455353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,8,8,128,1,float16,float16,0,0.05350080132484436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,8,8,128,1,float16,fp8,0,0.04716159999370575
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,8,1,128,1,float16,float16,0,0.03607200086116791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,8,2,128,1,fp8,fp8,0,0.01921280026435852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,8,1,128,1,fp8,fp8,0,0.03681440055370331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,8,2,128,1,float16,float16,0,0.038961601257324216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,8,2,128,1,float16,fp8,0,0.03705919981002807
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,8,2,128,1,fp8,fp8,0,0.03696160018444061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,8,4,128,1,float16,float16,0,0.04414399862289429
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,8,4,128,1,float16,fp8,0,0.0395440012216568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,8,4,128,1,fp8,fp8,0,0.038980799913406375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,8,1,128,1,float16,float16,0,0.11365760564804077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,8,8,128,1,float16,float16,0,0.031137600541114807
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,8,8,128,1,float16,fp8,0,0.025153601169586183
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,8,8,128,1,fp8,fp8,0,0.024987199902534486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,8,1,128,1,float16,float16,0,0.0215488001704216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,8,1,128,1,float16,fp8,0,0.023236800730228425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,8,1,128,1,fp8,fp8,0,0.02335200011730194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,8,1,128,1,float16,float16,0,0.06446239948272706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,8,2,128,1,float16,float16,0,0.021729600429534913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,8,2,128,1,float16,fp8,0,0.02335679978132248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,8,4,128,1,float16,float16,0,0.023236800730228425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,8,4,128,1,float16,fp8,0,0.02369759976863861
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,8,4,128,1,fp8,fp8,0,0.023715199530124666
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,8,2,128,1,fp8,fp8,0,0.01858399957418442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,8,8,128,1,float16,float16,0,0.017080000042915343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,8,8,128,1,float16,fp8,0,0.01652960032224655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,8,1,128,1,float16,float16,0,0.014764800667762756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,8,8,128,1,fp8,fp8,0,0.046793600916862486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,8,1,128,1,float16,fp8,0,0.015436799824237823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,8,1,128,1,fp8,fp8,0,0.015624000132083893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,8,2,128,1,float16,fp8,0,0.015471999347209931
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,8,2,128,1,fp8,fp8,0,0.015427200496196747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,8,4,128,1,float16,float16,0,0.015622399747371674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,8,4,128,1,float16,fp8,0,0.01579679995775223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,8,4,128,1,fp8,fp8,0,0.015830400586128234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,8,8,128,1,float16,float16,0,0.015457600355148315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,8,8,128,1,float16,fp8,0,0.015887999534606935
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,8,8,128,1,fp8,fp8,0,0.015971200168132783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,8,1,128,1,float16,float16,0,0.014312000572681427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,8,1,128,1,float16,fp8,0,0.015278400480747223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,8,1,128,1,fp8,fp8,0,0.015219199657440185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,8,2,128,1,float16,float16,0,0.014420799911022186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,8,2,128,1,float16,fp8,0,0.015108799934387207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,8,2,128,1,fp8,fp8,0,0.015094399452209473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,8,4,128,1,float16,float16,0,0.01478399932384491
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,8,4,128,1,float16,fp8,0,0.01536799967288971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,8,4,128,1,fp8,fp8,0,0.015302400290966033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,8,8,128,1,float16,float16,0,0.014793600142002105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,8,8,128,1,float16,fp8,0,0.015572799742221833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,8,8,128,1,fp8,fp8,0,0.015240000188350677
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,8,2,128,1,fp8,fp8,0,0.023177599906921385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,8,1,128,1,float16,float16,0,0.01404159963130951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,8,1,128,1,float16,fp8,0,0.015036800503730774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,8,2,128,1,float16,float16,0,0.014153599739074707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,8,2,128,1,float16,fp8,0,0.01480640023946762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,8,2,128,1,fp8,fp8,0,0.01499360054731369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,8,4,128,1,fp8,fp8,0,0.07251039743423462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,8,4,128,1,float16,float16,0,0.014312000572681427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,8,4,128,1,fp8,fp8,0,0.015113599598407745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,8,8,128,1,float16,float16,0,0.014401599764823914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,8,2,128,1,float16,float16,0,0.014951999485492706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,8,1,128,1,float16,fp8,0,0.03707680106163025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,8,1,128,1,float16,float16,0,0.01379680037498474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,8,1,128,1,float16,fp8,0,0.014865599572658539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,8,1,128,1,fp8,fp8,0,0.01462559998035431
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,8,2,128,1,float16,float16,0,0.013779200613498688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,8,2,128,1,float16,fp8,0,0.014643199741840363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,8,2,128,1,fp8,fp8,0,0.014683200418949128
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,8,4,128,1,float16,fp8,0,0.014886400103569031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,8,4,128,1,fp8,fp8,0,0.01485760062932968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,8,1,128,1,float16,float16,0,0.10664000511169433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,8,1,128,1,float16,fp8,0,0.11420960426330566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,8,1,128,1,fp8,fp8,0,0.11452159881591797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,8,2,128,1,float16,float16,0,0.11274880170822144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,8,2,128,1,float16,fp8,0,0.11668800115585327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,8,2,128,1,fp8,fp8,0,0.11609280109405518
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,8,1,128,1,fp8,fp8,0,0.01510079950094223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,8,4,128,1,float16,float16,0,0.13159359693527223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,8,4,128,1,float16,fp8,0,0.1256160020828247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,8,4,128,1,fp8,fp8,0,0.12603039741516114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,8,8,128,1,float16,float16,0,0.0919376015663147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,8,8,128,1,float16,fp8,0,0.07892959713935851
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,8,8,128,1,fp8,fp8,0,0.07895039916038513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,8,1,128,1,float16,float16,0,0.06155359745025635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,8,8,128,1,float16,fp8,0,0.015246400237083435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,8,1,128,1,float16,fp8,0,0.06238080263137817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,8,1,128,1,fp8,fp8,0,0.06266080141067505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,8,8,128,1,fp8,fp8,0,0.014998400211334228
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,8,2,128,1,float16,float16,0,0.06404160261154175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,8,2,128,1,float16,fp8,0,0.0625711977481842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,8,2,128,1,fp8,fp8,0,0.06280480027198791
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,8,4,128,1,float16,float16,0,0.07239519953727722
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,8,4,128,1,float16,fp8,0,0.0672976016998291
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,8,4,128,1,fp8,fp8,0,0.06714879870414733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,8,8,128,1,float16,float16,0,0.05103359818458557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,8,8,128,1,float16,fp8,0,0.04249599874019623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,8,8,128,1,fp8,fp8,0,0.04232800006866455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,8,1,128,1,float16,fp8,0,0.03242720067501068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,8,1,128,1,fp8,fp8,0,0.03253760039806366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,8,2,128,1,float16,float16,0,0.037145599722862244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,8,2,128,1,float16,fp8,0,0.03249120116233826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,8,2,128,1,fp8,fp8,0,0.03267680108547211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,8,4,128,1,float16,fp8,0,0.034822401404380796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,8,4,128,1,fp8,fp8,0,0.03440000116825104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,8,8,128,1,float16,float16,0,0.028984001278877257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,8,8,128,1,float16,fp8,0,0.02205760031938553
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,8,8,128,1,fp8,fp8,0,0.021620799601078034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,8,1,128,1,float16,float16,0,0.019380800426006317
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,8,1,128,1,float16,fp8,0,0.020377600193023683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,8,1,128,1,fp8,fp8,0,0.02043039947748184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,8,2,128,1,float16,float16,0,0.01930239945650101
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,8,2,128,1,float16,fp8,0,0.020193600654602052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,8,2,128,1,fp8,fp8,0,0.02022880017757416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,8,4,128,1,float16,float16,0,0.02077440023422241
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,8,4,128,1,fp8,fp8,0,0.13156640529632568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,8,4,128,1,float16,fp8,0,0.020425599813461304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,8,8,128,1,float16,float16,0,0.015771199762821198
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,8,4,128,1,fp8,fp8,0,0.02028000056743622
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,8,8,128,1,float16,fp8,0,0.014903999865055084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,8,8,128,1,fp8,fp8,0,0.014974400401115417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,8,1,128,1,float16,float16,0,0.013638399541378021
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,8,1,128,1,float16,fp8,0,0.014238399267196656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,8,1,128,1,fp8,fp8,0,0.014233599603176116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,8,2,128,1,float16,float16,0,0.013889600336551667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,8,2,128,1,float16,fp8,0,0.014164799451828003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,8,2,128,1,fp8,fp8,0,0.014107200503349304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,8,4,128,1,float16,float16,0,0.014553600549697876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,8,4,128,1,float16,fp8,0,0.014259199798107147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,8,4,128,1,fp8,fp8,0,0.014560000598430633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,8,8,128,1,float16,float16,0,0.013736000657081604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,8,8,128,1,fp8,fp8,0,0.016809600591659545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,8,8,128,1,float16,fp8,0,0.013865600526332855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,8,8,128,1,fp8,fp8,0,0.013648000359535218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,8,1,128,1,float16,float16,0,0.012984000146389008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,8,1,128,1,fp8,fp8,0,0.013371199369430542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,8,2,128,1,float16,float16,0,0.013019199669361114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,8,2,128,1,float16,fp8,0,0.013259199261665345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,8,2,128,1,fp8,fp8,0,0.013731199502944946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,8,4,128,1,float16,float16,0,0.013447999954223633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,8,4,128,1,float16,fp8,0,0.01363999992609024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,8,8,128,1,float16,float16,0,0.013212800025939941
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,8,8,128,1,float16,fp8,0,0.013348799943923951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,8,8,128,1,fp8,fp8,0,0.013352000713348388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,8,1,128,1,float16,float16,0,0.03198719918727875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,8,1,128,1,float16,fp8,0,0.012966400384902954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,8,1,128,1,fp8,fp8,0,0.013014400005340576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,8,4,128,1,float16,float16,0,0.013731199502944946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,8,2,128,1,float16,fp8,0,0.013247999548912048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,8,2,128,1,fp8,fp8,0,0.01329279989004135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,8,4,128,1,float16,float16,0,0.012913599610328674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,8,4,128,1,float16,fp8,0,0.013232000172138214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,8,4,128,1,fp8,fp8,0,0.013299199938774108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,8,8,128,1,float16,float16,0,0.01257600039243698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,8,8,128,1,float16,fp8,0,0.013014400005340576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,8,8,128,1,fp8,fp8,0,0.012759999930858612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,8,1,128,1,float16,float16,0,0.012083200365304947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,8,1,128,1,float16,fp8,0,0.01265919953584671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,8,1,128,1,fp8,fp8,0,0.012950399518013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,8,2,128,1,float16,float16,0,0.012216000258922577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,8,2,128,1,float16,fp8,0,0.01281599998474121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,8,2,128,1,fp8,fp8,0,0.01289760023355484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,8,4,128,1,float16,float16,0,0.012299200147390365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,8,4,128,1,float16,fp8,0,0.013118399679660797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,8,4,128,1,fp8,fp8,0,0.012831999361515046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,8,8,128,1,float16,float16,0,0.012185599654912949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,8,4,128,1,float16,fp8,0,0.015081599354743958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,8,8,128,1,fp8,fp8,0,0.012691199779510498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,8,1,128,1,float16,float16,0,0.012030400335788727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,8,1,128,1,float16,fp8,0,0.012763200700283051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,8,4,128,1,fp8,fp8,0,0.01361600011587143
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,8,1,128,1,fp8,fp8,0,0.012780800461769104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,8,2,128,1,float16,fp8,0,0.012870399653911591
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,8,2,128,1,fp8,fp8,0,0.01252640038728714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,8,4,128,1,float16,float16,0,0.011838400363922119
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,8,1,128,1,float16,float16,0,0.012532800436019897
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,8,4,128,1,float16,fp8,0,0.0124719999730587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,8,4,128,1,fp8,fp8,0,0.012529599666595458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,8,1,128,1,float16,float16,0,0.06149759888648987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,8,1,128,1,float16,fp8,0,0.06374880075454711
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,8,2,128,1,float16,float16,0,0.0123648002743721
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,8,2,128,1,float16,fp8,0,0.06396960020065308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,8,2,128,1,fp8,fp8,0,0.06387360095977783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,8,4,128,1,float16,float16,0,0.0742959976196289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,8,4,128,1,float16,fp8,0,0.06895679831504822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,8,4,128,1,fp8,fp8,0,0.06842399835586548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,8,8,128,1,float16,float16,0,0.06659520268440247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,8,8,128,1,float16,fp8,0,0.06090400218963623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,8,8,128,1,fp8,fp8,0,0.060971200466156006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,8,1,128,1,float16,float16,0,0.03237760066986084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,8,1,128,1,float16,fp8,0,0.03342239856719971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,8,1,128,1,fp8,fp8,0,0.033390399813652036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,8,2,128,1,float16,float16,0,0.037544000148773196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,8,2,128,1,float16,fp8,0,0.0338016003370285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,8,2,128,1,fp8,fp8,0,0.03348160088062287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,8,4,128,1,float16,float16,0,0.042049598693847653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,8,4,128,1,float16,fp8,0,0.03515360057353974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,8,4,128,1,fp8,fp8,0,0.035718399286270144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,8,8,128,1,float16,float16,0,0.03678719997406006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,8,8,128,1,fp8,fp8,0,0.031244799494743347
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,8,1,128,1,float16,float16,0,0.01974560022354126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,8,1,128,1,float16,fp8,0,0.02077440023422241
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,8,1,128,1,fp8,fp8,0,0.020849600434303284
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,8,1,128,1,float16,fp8,0,0.013483199477195739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,8,2,128,1,float16,float16,0,0.019593599438667297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,8,2,128,1,float16,fp8,0,0.020766399800777435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,8,2,128,1,fp8,fp8,0,0.0208064004778862
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,8,4,128,1,float16,float16,0,0.020843200385570526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,8,4,128,1,fp8,fp8,0,0.02080159932374954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,8,8,128,1,float16,float16,0,0.019392000138759614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,8,8,128,1,float16,fp8,0,0.01956319957971573
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,8,4,128,1,float16,float16,0,0.04139040112495422
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,8,1,128,1,fp8,fp8,0,0.06345120072364807
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,8,2,128,1,float16,float16,0,0.06492159962654113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,8,1,128,1,float16,float16,0,0.014019200205802917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,8,1,128,1,float16,fp8,0,0.014707200229167938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,8,1,128,1,fp8,fp8,0,0.014567999541759491
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,8,2,128,1,float16,float16,0,0.014401599764823914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,8,2,128,1,float16,fp8,0,0.01470080018043518
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,8,2,128,1,fp8,fp8,0,0.014711999893188476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,8,4,128,1,float16,float16,0,0.014839999377727509
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,8,4,128,1,float16,fp8,0,0.014902399480342865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,8,4,128,1,fp8,fp8,0,0.014980800449848175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,8,8,128,1,float16,float16,0,0.013558399677276612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,8,8,128,1,float16,fp8,0,0.013873599469661713
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,8,8,128,1,fp8,fp8,0,0.013793599605560303
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,8,1,128,1,float16,float16,0,0.012895999848842621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,8,1,128,1,float16,fp8,0,0.013391999900341034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,8,2,128,1,float16,float16,0,0.012884800136089326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,8,2,128,1,float16,fp8,0,0.013444800674915314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,8,2,128,1,fp8,fp8,0,0.013257600367069244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,8,4,128,1,float16,float16,0,0.013059200346469879
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,8,4,128,1,float16,fp8,0,0.013692800700664521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,8,4,128,1,fp8,fp8,0,0.013742400705814362
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,8,8,128,1,float16,float16,0,0.012967999279499053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,8,8,128,1,float16,fp8,0,0.013526399433612824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,8,8,128,1,fp8,fp8,0,0.013368000090122224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,8,8,128,1,float16,fp8,0,0.012639999389648438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,8,1,128,1,float16,float16,0,0.012812800705432892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,8,1,128,1,float16,fp8,0,0.013131199777126313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,8,1,128,1,fp8,fp8,0,0.013374400138854981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,8,2,128,1,float16,float16,0,0.012510399520397186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,8,4,128,1,float16,fp8,0,0.020851199328899384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,8,2,128,1,fp8,fp8,0,0.01327040046453476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,8,4,128,1,float16,float16,0,0.012862400710582733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,8,4,128,1,float16,fp8,0,0.013407999277114868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,8,2,128,1,float16,float16,0,0.012064000219106674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,8,8,128,1,float16,float16,0,0.012374400347471236
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,8,8,128,1,float16,fp8,0,0.013172799348831176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,8,8,128,1,fp8,fp8,0,0.013007999956607818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,8,1,128,1,float16,float16,0,0.012257599830627441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,8,1,128,1,float16,fp8,0,0.012859199941158295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,8,1,128,1,fp8,fp8,0,0.012667199969291687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,8,2,128,1,float16,float16,0,0.012275200337171555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,8,2,128,1,float16,fp8,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,8,2,128,1,fp8,fp8,0,0.013078400492668152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,8,4,128,1,float16,float16,0,0.012129600346088409
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,8,4,128,1,float16,fp8,0,0.012715199589729309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,8,4,128,1,fp8,fp8,0,0.012895999848842621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,8,8,128,1,float16,float16,0,0.011947199702262878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,8,8,128,1,float16,fp8,0,0.012564800679683685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,8,8,128,1,fp8,fp8,0,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,8,1,128,1,float16,float16,0,0.011980800330638886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,8,1,128,1,float16,fp8,0,0.012596799433231354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,8,1,128,1,fp8,fp8,0,0.012558400630950928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,8,2,128,1,float16,float16,0,0.012076800316572189
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,8,2,128,1,float16,fp8,0,0.012515200674533844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,8,2,128,1,fp8,fp8,0,0.012807999551296235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,8,4,128,1,float16,float16,0,0.011896000057458878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,8,4,128,1,float16,fp8,0,0.012807999551296235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,8,4,128,1,fp8,fp8,0,0.012535999715328216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,8,8,128,1,float16,float16,0,0.011905600130558015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,8,8,128,1,float16,fp8,0,0.012316799908876418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,8,8,128,1,fp8,fp8,0,0.0126351997256279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,8,1,128,1,float16,float16,0,0.011975999921560287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,8,1,128,1,float16,fp8,0,0.012736000120639801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,8,1,128,1,fp8,fp8,0,0.012612800300121307
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,8,2,128,1,float16,float16,0,0.011924800276756287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,8,2,128,1,float16,fp8,0,0.012587200105190276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,8,2,128,1,fp8,fp8,0,0.012558400630950928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,8,4,128,1,float16,float16,0,0.011902400106191636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,8,4,128,1,float16,fp8,0,0.012715199589729309
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,8,4,128,1,fp8,fp8,0,0.01257600039243698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,8,1,128,1,float16,float16,0,0.03361760079860687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,8,1,128,1,float16,fp8,0,0.03537920117378235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,8,1,128,1,fp8,fp8,0,0.035604798793792726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,8,2,128,1,float16,float16,0,0.03782399892807007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,8,8,128,1,float16,fp8,0,0.031142398715019226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,8,2,128,1,float16,fp8,0,0.035848000645637514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,8,2,128,1,fp8,fp8,0,0.03494719862937927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,8,4,128,1,float16,float16,0,0.057678401470184326
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,8,4,128,1,float16,fp8,0,0.05539199709892273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,8,8,128,1,float16,float16,0,0.052616000175476074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,8,8,128,1,float16,fp8,0,0.0498879998922348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,8,8,128,1,fp8,fp8,0,0.04975999891757965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,8,1,128,1,float16,float16,0,0.02017119973897934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,8,1,128,1,float16,fp8,0,0.02213920056819916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,8,2,128,1,float16,float16,0,0.02040960043668747
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,8,2,128,1,float16,fp8,0,0.022123199701309205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,8,2,128,1,float16,fp8,0,0.013318400084972381
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,8,4,128,1,float16,float16,0,0.028960001468658448
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,8,4,128,1,float16,fp8,0,0.03049120008945465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,8,4,128,1,fp8,fp8,0,0.030913600325584413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,8,4,128,1,fp8,fp8,0,0.013407999277114868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,8,8,128,1,float16,float16,0,0.02701759934425354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,8,8,128,1,fp8,fp8,0,0.02892799973487854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,8,1,128,1,float16,float16,0,0.014508800208568573
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,8,1,128,1,float16,fp8,0,0.015164799988269806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,8,1,128,1,fp8,fp8,0,0.015465599298477174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,8,2,128,1,float16,float16,0,0.014616000652313232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,8,2,128,1,float16,fp8,0,0.015353600680828094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,8,2,128,1,fp8,fp8,0,0.015033599734306336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,8,4,128,1,float16,float16,0,0.018408000469207764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,8,4,128,1,float16,fp8,0,0.019387200474739075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,8,4,128,1,fp8,fp8,0,0.01950239986181259
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,8,8,128,1,float16,float16,0,0.017870399355888366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,8,8,128,1,float16,fp8,0,0.01860480010509491
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,8,8,128,1,fp8,fp8,0,0.018822400271892546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,8,1,128,1,float16,float16,0,0.013665600121021271
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,8,1,128,1,float16,fp8,0,0.014030399918556213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,8,1,128,1,fp8,fp8,0,0.013356800377368926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,8,2,128,1,float16,float16,0,0.013206399977207184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,8,2,128,1,float16,fp8,0,0.013831999897956849
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,8,2,128,1,fp8,fp8,0,0.013799999654293061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,8,4,128,1,float16,float16,0,0.013352000713348388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,8,4,128,1,float16,fp8,0,0.013991999626159667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,8,4,128,1,fp8,fp8,0,0.01401119977235794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,8,4,128,1,fp8,fp8,0,0.055062401294708255
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,8,8,128,1,float16,fp8,0,0.013407999277114868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,8,8,128,1,fp8,fp8,0,0.013507199287414551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,8,1,128,1,float16,float16,0,0.013123199343681335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,8,1,128,1,fp8,fp8,0,0.021723200380802155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,8,1,128,1,float16,fp8,0,0.013740800321102142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,8,2,128,1,fp8,fp8,0,0.02179519981145859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,8,1,128,1,fp8,fp8,0,0.013376000523567199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,8,2,128,1,float16,float16,0,0.013064000010490417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,8,2,128,1,float16,fp8,0,0.013374400138854981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,8,2,128,1,fp8,fp8,0,0.01345279961824417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,8,4,128,1,float16,fp8,0,0.013406400382518769
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,8,4,128,1,fp8,fp8,0,0.013305599987506866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,8,8,128,1,float16,float16,0,0.012484800070524216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,8,8,128,1,float16,fp8,0,0.013247999548912048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,8,8,128,1,fp8,fp8,0,0.013142399489879608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,8,1,128,1,float16,float16,0,0.012492799758911132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,8,1,128,1,float16,fp8,0,0.013313600420951843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,8,1,128,1,fp8,fp8,0,0.013347199559211731
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,8,2,128,1,float16,float16,0,0.012220799922943115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,8,2,128,1,float16,fp8,0,0.012934400141239167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,8,2,128,1,fp8,fp8,0,0.01273760050535202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,8,4,128,1,float16,float16,0,0.012263999879360199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,8,4,128,1,float16,fp8,0,0.01281760036945343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,8,4,128,1,fp8,fp8,0,0.01287200003862381
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,8,8,128,1,fp8,fp8,0,0.01920959949493408
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,8,8,128,1,float16,fp8,0,0.012718400359153748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,8,8,128,1,fp8,fp8,0,0.012721599638462066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,8,1,128,1,float16,fp8,0,0.012761600315570831
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,8,1,128,1,fp8,fp8,0,0.012750400602817536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,8,2,128,1,float16,float16,0,0.01213119998574257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,8,2,128,1,float16,fp8,0,0.012716799974441528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,8,2,128,1,fp8,fp8,0,0.01265760064125061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,8,4,128,1,float16,float16,0,0.011764799803495407
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,8,4,128,1,float16,fp8,0,0.012921600043773651
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,8,4,128,1,fp8,fp8,0,0.012638400495052337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,8,8,128,1,float16,float16,0,0.011857599765062333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,8,8,128,1,float16,fp8,0,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,8,8,128,1,fp8,fp8,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,8,1,128,1,float16,float16,0,0.011849600076675414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,8,1,128,1,float16,fp8,0,0.012532800436019897
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,8,1,128,1,fp8,fp8,0,0.01252799928188324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,8,8,128,1,float16,float16,0,0.013184000551700593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,8,2,128,1,float16,fp8,0,0.012545600533485413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,8,2,128,1,fp8,fp8,0,0.012454400211572647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,8,4,128,1,float16,float16,0,0.011611200124025344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,8,4,128,1,float16,fp8,0,0.012439999729394913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,8,4,128,1,fp8,fp8,0,0.012459199875593185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,8,8,128,1,float16,float16,0,0.011769600212574005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,8,4,128,1,float16,float16,0,0.012559999525547028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,8,8,128,1,float16,fp8,0,0.012439999729394913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,8,8,128,1,fp8,fp8,0,0.012247999757528305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,8,1,128,1,float16,float16,0,0.011894399672746659
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,8,1,128,1,float16,fp8,0,0.012908799946308136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,8,2,128,1,float16,float16,0,0.011856000125408172
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,8,2,128,1,float16,fp8,0,0.01255200058221817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,8,2,128,1,fp8,fp8,0,0.0125231996178627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,8,4,128,1,float16,float16,0,0.011671999841928482
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,8,8,128,1,float16,float16,0,0.011998400092124939
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,8,4,128,1,float16,fp8,0,0.012347199767827988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,8,4,128,1,fp8,fp8,0,0.0123648002743721
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,8,1,128,1,float16,float16,0,0.012176000326871873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,8,1,128,1,float16,float16,0,0.022419199347496033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,8,1,128,1,fp8,fp8,0,0.024585600197315215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,8,2,128,1,float16,float16,0,0.0296640008687973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,8,2,128,1,fp8,fp8,0,0.03312639892101288
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,8,4,128,1,float16,float16,0,0.04594079852104187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,8,4,128,1,fp8,fp8,0,0.05085279941558838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,8,8,128,1,float16,float16,0,0.04256480038166046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,8,1,128,1,fp8,fp8,0,0.014071999490261078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,8,8,128,1,float16,fp8,0,0.04795520007610321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,8,8,128,1,fp8,fp8,0,0.04781759977340698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,8,1,128,1,float16,float16,0,0.015516799688339234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,8,1,128,1,float16,fp8,0,0.016305600106716157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,8,1,128,1,fp8,fp8,0,0.016223999857902526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,8,2,128,1,float16,float16,0,0.01908639967441559
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,8,2,128,1,float16,fp8,0,0.020483200252056123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,8,2,128,1,fp8,fp8,0,0.020640000700950623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,8,4,128,1,float16,float16,0,0.027136000990867614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,8,4,128,1,float16,fp8,0,0.029841598868370057
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,8,4,128,1,fp8,fp8,0,0.029662400484085083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,8,8,128,1,float16,float16,0,0.025972801446914672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,8,2,128,1,float16,float16,0,0.011844799667596818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,8,8,128,1,fp8,fp8,0,0.028313601016998292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,8,1,128,1,float16,float16,0,0.013788799941539764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,8,1,128,1,float16,fp8,0,0.01446399986743927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,8,1,128,1,fp8,fp8,0,0.0147024005651474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,8,2,128,1,float16,float16,0,0.013752000033855438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,8,2,128,1,float16,fp8,0,0.014523200690746307
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,8,2,128,1,fp8,fp8,0,0.01451359987258911
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,8,8,128,1,float16,fp8,0,0.02894560098648071
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,8,4,128,1,float16,float16,0,0.017841599881649017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,8,1,128,1,fp8,fp8,0,0.012641599774360657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,8,4,128,1,float16,fp8,0,0.019385600090026857
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,8,4,128,1,fp8,fp8,0,0.018931199610233308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,8,8,128,1,float16,float16,0,0.01725279986858368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,8,8,128,1,float16,fp8,0,0.018352000415325163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,8,1,128,1,float16,float16,0,0.013291199505329133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,8,1,128,1,float16,fp8,0,0.013782399892807006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,8,1,128,1,fp8,fp8,0,0.013995200395584106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,8,2,128,1,float16,float16,0,0.012836800515651703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,8,2,128,1,float16,fp8,0,0.013550400733947754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,8,4,128,1,float16,float16,0,0.013371199369430542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,8,1,128,1,float16,fp8,0,0.024460799992084503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,8,4,128,1,float16,fp8,0,0.014025600254535675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,8,4,128,1,fp8,fp8,0,0.013780799508094788
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,8,8,128,1,float16,float16,0,0.012833599746227265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,8,8,128,1,float16,fp8,0,0.013264000415802002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,8,4,128,1,float16,fp8,0,0.05102720260620117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,8,1,128,1,float16,fp8,0,0.012807999551296235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,8,1,128,1,fp8,fp8,0,0.01282079964876175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,8,2,128,1,float16,float16,0,0.012185599654912949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,8,2,128,1,float16,fp8,0,0.012963199615478515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,8,2,128,1,fp8,fp8,0,0.012958399951457977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,8,4,128,1,float16,float16,0,0.012372799962759019
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,8,4,128,1,float16,fp8,0,0.013046400249004364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,8,4,128,1,fp8,fp8,0,0.013007999956607818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,8,8,128,1,float16,float16,0,0.012307199835777282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,8,8,128,1,float16,fp8,0,0.012913599610328674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,8,8,128,1,float16,fp8,0,0.028271999955177308
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,8,1,128,1,float16,float16,0,0.012307199835777282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,8,1,128,1,float16,fp8,0,0.012956799566745758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,8,1,128,1,fp8,fp8,0,0.012969599664211273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,8,2,128,1,float16,float16,0,0.01199359968304634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,8,2,128,1,float16,fp8,0,0.012649600207805634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,8,2,128,1,fp8,fp8,0,0.012651200592517852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,8,4,128,1,float16,float16,0,0.012160000205039979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,8,4,128,1,float16,fp8,0,0.012868799269199371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,8,4,128,1,fp8,fp8,0,0.012875199317932129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,8,8,128,1,float16,float16,0,0.01199679970741272
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,8,8,128,1,fp8,fp8,0,0.018156799674034118
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,8,8,128,1,float16,fp8,0,0.012763200700283051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,8,8,128,1,fp8,fp8,0,0.012913599610328674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,8,1,128,1,float16,fp8,0,0.012788799405097962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,8,2,128,1,fp8,fp8,0,0.013777600228786468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,8,1,128,1,fp8,fp8,0,0.01279360055923462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,8,2,128,1,float16,float16,0,0.011897599697113037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,8,2,128,1,fp8,fp8,0,0.012652799487113953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,8,2,128,1,float16,fp8,0,0.03253600001335144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,8,4,128,1,float16,float16,0,0.0118928000330925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,8,8,128,1,fp8,fp8,0,0.013484799861907959
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,8,4,128,1,fp8,fp8,0,0.012636800110340119
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,8,1,128,1,float16,float16,0,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,8,8,128,1,float16,float16,0,0.011803200095891952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,8,8,128,1,float16,fp8,0,0.012436799705028534
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,8,8,128,1,fp8,fp8,0,0.012368000298738479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,8,1,128,1,float16,float16,0,0.011915200203657151
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,8,1,128,1,float16,fp8,0,0.0126351997256279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,8,1,128,1,fp8,fp8,0,0.012590399384498597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,8,2,128,1,float16,fp8,0,0.012318400293588638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,8,2,128,1,fp8,fp8,0,0.012425599992275238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,8,4,128,1,float16,float16,0,0.011635199934244157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,8,4,128,1,float16,fp8,0,0.01244639977812767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,8,4,128,1,fp8,fp8,0,0.012352000176906585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,8,8,128,1,float16,float16,0,0.011587200313806533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,8,8,128,1,float16,fp8,0,0.012399999797344208
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,8,8,128,1,fp8,fp8,0,0.012345600128173827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,8,1,128,1,float16,float16,0,0.011883199959993363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,8,1,128,1,float16,fp8,0,0.012275200337171555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,8,1,128,1,fp8,fp8,0,0.012432000041007996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,8,2,128,1,float16,float16,0,0.011777599900960922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,8,2,128,1,float16,fp8,0,0.012139199674129486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,8,2,128,1,fp8,fp8,0,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,8,4,128,1,float16,float16,0,0.011791999638080596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,8,4,128,1,float16,fp8,0,0.012363199889659882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,8,4,128,1,fp8,fp8,0,0.012265600264072418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,8,1,128,1,float16,float16,0,0.02086080014705658
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,8,1,128,1,float16,fp8,0,0.022995199263095855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,8,2,128,1,float16,float16,0,0.02826719880104065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,8,8,128,1,fp8,fp8,0,0.012934400141239167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,8,2,128,1,fp8,fp8,0,0.031036800146102904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,8,4,128,1,float16,float16,0,0.04298079907894135
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,8,4,128,1,float16,fp8,0,0.048879998922348025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,8,4,128,1,fp8,fp8,0,0.048923200368881224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,8,8,128,1,float16,fp8,0,0.04692479968070984
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,8,8,128,1,fp8,fp8,0,0.04697119891643524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,8,1,128,1,float16,float16,0,0.01517920047044754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,8,1,128,1,float16,fp8,0,0.0159183993935585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,8,1,128,1,fp8,fp8,0,0.01590079963207245
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,8,2,128,1,float16,float16,0,0.018697600066661834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,8,2,128,1,float16,fp8,0,0.020113599300384522
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,8,2,128,1,fp8,fp8,0,0.02014079988002777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,8,4,128,1,float16,float16,0,0.026183998584747313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,8,4,128,1,float16,fp8,0,0.029232001304626463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,8,2,128,1,float16,fp8,0,0.012483199685811996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,8,4,128,1,fp8,fp8,0,0.02927359938621521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,8,8,128,1,float16,float16,0,0.02471359968185425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,8,4,128,1,float16,fp8,0,0.012464000284671784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,8,8,128,1,float16,fp8,0,0.027580800652503967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,8,8,128,1,fp8,fp8,0,0.027531200647354127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,8,1,128,1,float16,float16,0,0.013553600013256072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,8,2,128,1,float16,float16,0,0.01159999966621399
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,8,2,128,1,float16,float16,0,0.013331200182437896
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,8,2,128,1,float16,fp8,0,0.014238399267196656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,8,2,128,1,fp8,fp8,0,0.014291200041770934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,8,4,128,1,float16,float16,0,0.01719360053539276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,8,4,128,1,float16,fp8,0,0.018568000197410582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,8,4,128,1,fp8,fp8,0,0.018534399569034576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,8,1,128,1,fp8,fp8,0,0.022275200486183165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,8,2,128,1,float16,fp8,0,0.031574401259422305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,8,8,128,1,fp8,fp8,0,0.017871999740600587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,8,1,128,1,float16,float16,0,0.012356799840927125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,8,8,128,1,float16,float16,0,0.04061119854450226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,8,1,128,1,float16,fp8,0,0.013196800649166108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,8,1,128,1,fp8,fp8,0,0.01327040046453476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,8,2,128,1,float16,fp8,0,0.013491199910640716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,8,2,128,1,fp8,fp8,0,0.013436800241470337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,8,4,128,1,float16,float16,0,0.012588800489902496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,8,4,128,1,float16,fp8,0,0.013478399813175201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,8,4,128,1,fp8,fp8,0,0.013502399623394012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,8,8,128,1,float16,float16,0,0.012334399670362473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,8,8,128,1,float16,fp8,0,0.013257600367069244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,8,1,128,1,float16,float16,0,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,8,8,128,1,fp8,fp8,0,0.013283200562000275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,8,1,128,1,float16,float16,0,0.011720000207424164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,8,1,128,1,float16,fp8,0,0.012670400738716125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,8,1,128,1,fp8,fp8,0,0.012505599856376648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,8,2,128,1,float16,fp8,0,0.012702399492263794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,8,2,128,1,fp8,fp8,0,0.012692800164222718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,8,4,128,1,float16,float16,0,0.012225600332021714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,8,4,128,1,float16,fp8,0,0.012910400331020356
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,8,4,128,1,fp8,fp8,0,0.012936000525951386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,8,1,128,1,float16,fp8,0,0.014539200067520141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,8,1,128,1,fp8,fp8,0,0.014364799857139588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,8,8,128,1,fp8,fp8,0,0.012895999848842621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,8,1,128,1,float16,float16,0,0.011793600022792816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,8,1,128,1,float16,fp8,0,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,8,1,128,1,fp8,fp8,0,0.01260959953069687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,8,2,128,1,float16,float16,0,0.011958400160074234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,8,2,128,1,float16,fp8,0,0.012665599584579468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,8,2,128,1,fp8,fp8,0,0.01268640011548996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,8,4,128,1,float16,float16,0,0.011902400106191636
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,8,8,128,1,float16,fp8,0,0.017726400494575502
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,8,4,128,1,float16,fp8,0,0.012694400548934937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,8,4,128,1,fp8,fp8,0,0.012700800597667695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,8,8,128,1,float16,float16,0,0.011851199716329575
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,8,8,128,1,float16,fp8,0,0.012670400738716125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,8,8,128,1,fp8,fp8,0,0.01271039992570877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,8,1,128,1,float16,float16,0,0.011503999680280685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,8,1,128,1,float16,fp8,0,0.0123648002743721
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,8,2,128,1,float16,float16,0,0.012467200309038163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,8,2,128,1,float16,float16,0,0.011631999909877778
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,8,2,128,1,float16,fp8,0,0.012094400078058242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,8,4,128,1,float16,float16,0,0.011667200177907944
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,8,4,128,1,float16,fp8,0,0.01234399974346161
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,8,4,128,1,fp8,fp8,0,0.012377600371837615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,8,8,128,1,float16,float16,0,0.011657600104808808
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,8,8,128,1,float16,fp8,0,0.012433599680662155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,8,8,128,1,fp8,fp8,0,0.012409599870443344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,8,1,128,1,float16,float16,0,0.011217600107192994
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,8,2,128,1,float16,float16,0,0.012080000340938568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,8,1,128,1,fp8,fp8,0,0.01231359988451004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,8,1,128,1,float16,fp8,0,0.011924800276756287
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,8,2,128,1,float16,fp8,0,0.012191999703645706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,8,2,128,1,fp8,fp8,0,0.012004800140857697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,8,4,128,1,float16,fp8,0,0.012078399956226348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,8,4,128,1,fp8,fp8,0,0.012055999785661697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,8,8,128,1,float16,float16,0,0.01149120032787323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,8,8,128,1,float16,fp8,0,0.01295199990272522
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,8,8,128,1,float16,fp8,0,0.012294399738311767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,8,8,128,1,fp8,fp8,0,0.012240000069141388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,8,1,128,1,float16,float16,0,0.011388800293207168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,8,1,128,1,float16,fp8,0,0.012167999893426895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,8,1,128,1,fp8,fp8,0,0.012123200297355651
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,8,2,128,1,float16,float16,0,0.011292800307273865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,8,2,128,1,float16,fp8,0,0.012118399888277055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,8,2,128,1,fp8,fp8,0,0.011870399862527848
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,8,4,128,1,float16,float16,0,0.011350400000810622
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,8,4,128,1,float16,fp8,0,0.012015999853610992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,8,4,128,1,fp8,fp8,0,0.01207360029220581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,8,8,128,1,float16,float16,0,0.016518400609493257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,8,1,128,1,fp8,fp8,0,0.012256000190973282
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,8,2,128,1,fp8,fp8,0,0.01228479966521263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,8,2,128,1,float16,float16,0,0.011601600050926208
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,8,4,128,1,float16,float16,0,0.011512000113725662
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,8,8,128,1,float16,float16,0,0.012094400078058242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,4,1,128,1,float16,fp8,0,3.196811294555664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,4,1,128,1,fp8,fp8,0,3.1573823928833007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,4,1,128,1,float16,float16,0,3.758884811401367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,4,2,128,1,float16,fp8,0,3.215865707397461
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,4,4,128,1,float16,float16,0,1.7506975173950194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,4,4,128,1,float16,fp8,0,1.6282896041870116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,4,2,128,1,float16,float16,0,3.7397552490234376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,4,4,128,1,fp8,fp8,0,1.6562591552734376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,4,1,128,1,float16,float16,0,1.7788911819458009
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,4,1,128,1,float16,fp8,0,1.7784927368164063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,4,2,128,1,fp8,fp8,0,3.1991455078125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,4,1,128,1,fp8,fp8,0,1.6244592666625977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,4,2,128,1,float16,fp8,0,1.593062400817871
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,4,2,128,1,float16,float16,0,1.8009952545166015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,4,4,128,1,float16,float16,0,0.9393168449401855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,4,4,128,1,float16,fp8,0,0.853451156616211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,4,4,128,1,fp8,fp8,0,0.8520048141479493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,4,1,128,1,float16,float16,0,0.8557632446289063
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,4,1,128,1,float16,fp8,0,0.8085040092468262
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,4,1,128,1,fp8,fp8,0,0.9248271942138672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,4,2,128,1,float16,fp8,0,0.8182479858398437
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,4,4,128,1,float16,float16,0,0.4537487983703613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,4,2,128,1,fp8,fp8,0,0.8213264465332031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,4,4,128,1,float16,fp8,0,0.4260672092437744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,4,4,128,1,fp8,fp8,0,0.460038423538208
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,4,1,128,1,float16,float16,0,0.438702392578125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,4,1,128,1,float16,fp8,0,0.42325282096862793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,4,2,128,1,float16,float16,0,0.43835201263427737
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,4,2,128,1,float16,fp8,0,0.43075361251831057
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,4,2,128,1,fp8,fp8,0,0.408735990524292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,4,1,128,1,float16,float16,0,1.951982307434082
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,4,2,128,1,fp8,fp8,0,1.7555152893066406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,4,1,128,1,fp8,fp8,0,1.87216796875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,4,2,128,1,float16,float16,0,2.1116735458374025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,4,2,128,1,float16,float16,0,0.9017791748046875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,4,4,128,1,float16,float16,0,1.0614383697509766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,4,1,128,1,fp8,fp8,0,0.40436320304870604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,4,4,128,1,float16,fp8,0,1.085416030883789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,4,2,128,1,fp8,fp8,0,1.8361631393432618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,4,4,128,1,fp8,fp8,0,0.9601743698120118
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,4,2,128,1,float16,fp8,0,1.9808687210083007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,4,1,128,1,float16,float16,0,0.9786543846130371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,4,1,128,1,float16,fp8,0,0.925499153137207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,4,1,128,1,fp8,fp8,0,0.9318495750427246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,4,4,128,1,float16,float16,0,0.5746831893920898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,4,2,128,1,float16,float16,0,1.0516703605651856
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,4,2,128,1,float16,fp8,0,0.9270159721374511
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,4,4,128,1,float16,fp8,0,0.4973951816558838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,4,4,128,1,fp8,fp8,0,0.5012207984924316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,4,1,128,1,float16,float16,0,0.49793438911437987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,4,1,128,1,float16,fp8,0,0.4952047824859619
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,4,1,128,1,fp8,fp8,0,0.5068336009979248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,4,2,128,1,float16,fp8,0,0.46668481826782227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,4,1,128,1,float16,fp8,0,1.8360383987426758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,4,4,128,1,float16,fp8,0,0.28009920120239257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,4,4,128,1,fp8,fp8,0,0.28124001026153567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,4,1,128,1,float16,float16,0,0.27177441120147705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,4,1,128,1,float16,fp8,0,0.2548127889633179
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,4,1,128,1,fp8,fp8,0,0.2696991920471191
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,4,2,128,1,float16,float16,0,0.2827039957046509
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,4,2,128,1,float16,fp8,0,0.25999040603637696
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,4,2,128,1,fp8,fp8,0,0.25519518852233886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,4,1,128,1,float16,fp8,0,1.3166144371032715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,4,1,128,1,fp8,fp8,0,1.2843487739562989
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,4,2,128,1,float16,float16,0,1.4308783531188964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,4,2,128,1,fp8,fp8,0,0.9278703689575195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,4,4,128,1,float16,float16,0,0.8017807960510254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,4,2,128,1,float16,fp8,0,1.2943471908569335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,4,2,128,1,fp8,fp8,0,1.2856207847595216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,4,2,128,1,float16,float16,0,0.517622423171997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,4,4,128,1,float16,float16,0,0.31150720119476316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,4,4,128,1,float16,fp8,0,0.6900144100189209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,4,4,128,1,fp8,fp8,0,0.7402639865875245
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,4,1,128,1,float16,float16,0,0.7070303916931152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,4,1,128,1,float16,fp8,0,0.6571536064147949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,4,1,128,1,fp8,fp8,0,0.6577712059020996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,4,2,128,1,float16,float16,0,0.7226064205169678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,4,2,128,1,float16,fp8,0,0.6588719844818115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,4,4,128,1,float16,fp8,0,0.363044810295105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,4,4,128,1,fp8,fp8,0,0.36281120777130127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,4,1,128,1,float16,float16,0,0.3623807907104492
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,4,1,128,1,float16,fp8,0,0.33775360584259034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,4,1,128,1,fp8,fp8,0,0.33729920387268064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,4,2,128,1,float16,float16,0,0.37237598896026614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,4,2,128,1,float16,fp8,0,0.3376768112182617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,4,2,128,1,fp8,fp8,0,0.3391551971435547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,4,4,128,1,float16,fp8,0,0.18706879615783692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,4,1,128,1,float16,float16,0,1.4303919792175293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,4,4,128,1,fp8,fp8,0,0.18801920413970946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,4,1,128,1,float16,float16,0,0.19372639656066895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,4,1,128,1,float16,fp8,0,0.18194080591201783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,4,1,128,1,fp8,fp8,0,0.18510880470275878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,4,2,128,1,float16,float16,0,0.19695520401000977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,4,2,128,1,float16,fp8,0,0.1840016007423401
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,4,2,128,1,fp8,fp8,0,0.1823359966278076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,4,2,128,1,fp8,fp8,0,0.4759632110595703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,4,1,128,1,float16,fp8,0,1.6556896209716796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,4,1,128,1,float16,float16,0,1.8405872344970704
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,4,1,128,1,fp8,fp8,0,1.6799455642700196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,4,2,128,1,float16,float16,0,1.8282495498657227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,4,4,128,1,float16,float16,0,0.4206863880157471
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,4,2,128,1,fp8,fp8,0,0.7100671768188477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,4,4,128,1,float16,fp8,0,0.8675711631774903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,4,2,128,1,float16,fp8,0,1.6858047485351562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,4,4,128,1,float16,float16,0,0.24345118999481202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,4,4,128,1,fp8,fp8,0,0.9980959892272949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,4,1,128,1,float16,float16,0,0.9140416145324707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,4,1,128,1,float16,fp8,0,0.8470463752746582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,4,2,128,1,fp8,fp8,0,1.6665376663208007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,4,4,128,1,float16,float16,0,0.4849584102630615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,4,2,128,1,float16,float16,0,0.9175104141235352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,4,2,128,1,float16,fp8,0,0.8427040100097656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,4,4,128,1,float16,fp8,0,0.45238242149353025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,4,2,128,1,fp8,fp8,0,0.8971023559570312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,4,4,128,1,fp8,fp8,0,0.44270877838134765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,4,1,128,1,float16,fp8,0,0.4264063835144043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,4,1,128,1,fp8,fp8,0,0.43326401710510254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,4,2,128,1,float16,float16,0,0.4646927833557129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,4,2,128,1,float16,fp8,0,0.43166399002075195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,4,2,128,1,fp8,fp8,0,0.42879037857055663
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,4,4,128,1,fp8,fp8,0,0.2276047945022583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,4,1,128,1,float16,float16,0,0.23068640232086182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,4,1,128,1,float16,fp8,0,0.21475520133972167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,4,1,128,1,fp8,fp8,0,0.21559839248657225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,4,2,128,1,float16,float16,0,0.23808319568634034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,4,2,128,1,float16,fp8,0,0.21823520660400392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,4,2,128,1,fp8,fp8,0,0.2168800115585327
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,4,4,128,1,float16,float16,0,0.12539360523223878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,4,4,128,1,float16,float16,0,1.0052783966064454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,4,4,128,1,fp8,fp8,0,0.11702239513397217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,4,1,128,1,float16,float16,0,0.11803359985351562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,4,1,128,1,fp8,fp8,0,0.11523679494857789
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,4,2,128,1,float16,float16,0,0.11838239431381226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,4,1,128,1,fp8,fp8,0,0.8417584419250488
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,4,2,128,1,float16,fp8,0,0.11486719846725464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,4,2,128,1,fp8,fp8,0,0.11325600147247314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,4,1,128,1,float16,float16,0,0.4781360149383545
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,4,4,128,1,float16,float16,0,0.24600000381469728
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,4,1,128,1,float16,float16,0,1.0500176429748536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,4,1,128,1,float16,fp8,0,0.9815919876098633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,4,1,128,1,fp8,fp8,0,0.989568042755127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,4,2,128,1,float16,float16,0,1.0654159545898438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,4,2,128,1,float16,fp8,0,1.059928035736084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,4,2,128,1,fp8,fp8,0,0.985865592956543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,4,4,128,1,float16,fp8,0,0.11757919788360596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,4,4,128,1,float16,fp8,0,0.5403567790985108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,4,4,128,1,fp8,fp8,0,0.5470816135406494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,4,1,128,1,float16,fp8,0,0.5025487899780273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,4,1,128,1,fp8,fp8,0,0.5060351848602295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,4,2,128,1,float16,float16,0,0.5445104122161866
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,4,2,128,1,fp8,fp8,0,0.5049359798431396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,4,4,128,1,float16,float16,0,0.3116719961166382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,4,4,128,1,float16,fp8,0,0.2776576042175293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,4,4,128,1,fp8,fp8,0,0.2698415994644165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,4,1,128,1,float16,float16,0,0.2781008005142212
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,4,1,128,1,float16,fp8,0,0.26863200664520265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,4,1,128,1,fp8,fp8,0,0.26262080669403076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,4,2,128,1,float16,float16,0,0.2837232112884521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,4,4,128,1,float16,fp8,0,0.23721439838409425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,4,2,128,1,float16,fp8,0,0.27019519805908204
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,4,2,128,1,fp8,fp8,0,0.26184639930725095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,4,4,128,1,float16,float16,0,0.17185120582580565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,4,4,128,1,float16,fp8,0,0.14434399604797363
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,4,4,128,1,fp8,fp8,0,0.14904799461364746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,4,1,128,1,float16,float16,0,0.14577440023422242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,4,1,128,1,float16,fp8,0,0.14053280353546144
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,4,1,128,1,fp8,fp8,0,0.14070240259170533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,4,2,128,1,float16,float16,0,0.15503519773483276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,4,2,128,1,float16,fp8,0,0.1400063991546631
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,4,2,128,1,fp8,fp8,0,0.1417248010635376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,4,4,128,1,float16,fp8,0,0.09144160151481628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,4,4,128,1,fp8,fp8,0,0.08752639889717102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,4,1,128,1,float16,float16,0,0.09032959938049316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,4,1,128,1,float16,fp8,0,0.08955680131912232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,4,1,128,1,fp8,fp8,0,0.0863215982913971
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,4,2,128,1,float16,float16,0,0.090692800283432
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,4,2,128,1,float16,fp8,0,0.08942880034446717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,4,2,128,1,fp8,fp8,0,0.08661280274391174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,4,4,128,1,float16,float16,0,0.5803584098815918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,4,1,128,1,float16,fp8,0,0.11287519931793213
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,4,1,128,1,float16,float16,0,0.5270607948303223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,4,1,128,1,float16,float16,0,0.9898063659667968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,4,1,128,1,float16,fp8,0,0.93603515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,4,1,128,1,fp8,fp8,0,0.9279520034790039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,4,2,128,1,float16,float16,0,0.9965760231018066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,4,2,128,1,float16,fp8,0,0.517195177078247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,4,4,128,1,float16,fp8,0,0.49564480781555176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,4,2,128,1,float16,fp8,0,0.9335904121398926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,4,2,128,1,fp8,fp8,0,0.9342096328735352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,4,1,128,1,float16,float16,0,0.4900959968566895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,4,1,128,1,float16,fp8,0,0.47700161933898927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,4,1,128,1,fp8,fp8,0,0.472321605682373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,4,2,128,1,float16,float16,0,0.5043439865112305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,4,4,128,1,float16,float16,0,0.2765039920806885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,4,2,128,1,float16,fp8,0,0.47455520629882814
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,4,2,128,1,fp8,fp8,0,0.5014495849609375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,4,4,128,1,float16,fp8,0,0.253603196144104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,4,4,128,1,fp8,fp8,0,0.2529871940612793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,4,1,128,1,float16,float16,0,0.2514703989028931
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,4,1,128,1,float16,fp8,0,0.24341280460357667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,4,1,128,1,fp8,fp8,0,0.2517568111419678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,4,2,128,1,float16,float16,0,0.2593440055847168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,4,2,128,1,float16,fp8,0,0.24405279159545898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,4,4,128,1,float16,float16,0,0.09364320039749145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,4,4,128,1,fp8,fp8,0,0.13349440097808837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,4,1,128,1,float16,float16,0,0.12969919443130493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,4,1,128,1,fp8,fp8,0,0.12391680479049683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,4,2,128,1,float16,float16,0,0.13371679782867432
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,4,2,128,1,float16,fp8,0,0.12557599544525147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,4,2,128,1,fp8,fp8,0,0.12526079416275024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,4,4,128,1,float16,float16,0,0.07599840164184571
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,4,4,128,1,float16,fp8,0,0.06857119798660279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,4,4,128,1,fp8,fp8,0,0.0688207983970642
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,4,1,128,1,float16,float16,0,0.06826400160789489
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,4,1,128,1,float16,fp8,0,0.06737279891967773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,4,1,128,1,fp8,fp8,0,0.06726239919662476
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,4,2,128,1,float16,float16,0,0.06947519779205322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,4,2,128,1,float16,fp8,0,0.06791679859161377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,4,2,128,1,fp8,fp8,0,0.06746399998664857
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,4,4,128,1,float16,float16,0,0.06380640268325806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,4,4,128,1,float16,fp8,0,0.06139519810676575
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,4,4,128,1,fp8,fp8,0,0.061819201707839964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,4,1,128,1,float16,float16,0,0.0624351978302002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,4,1,128,1,float16,fp8,0,0.06133599877357483
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,4,1,128,1,fp8,fp8,0,0.061561602354049685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,4,4,128,1,float16,float16,0,0.144924795627594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,4,2,128,1,fp8,fp8,0,0.24606080055236818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,4,2,128,1,float16,fp8,0,0.06171360015869141
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,4,2,128,1,fp8,fp8,0,0.061956799030303954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,4,1,128,1,float16,float16,0,0.6306623935699462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,4,1,128,1,fp8,fp8,0,0.6093664169311523
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,4,4,128,1,fp8,fp8,0,0.491864013671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,4,2,128,1,float16,float16,0,0.6502255916595459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,4,2,128,1,float16,fp8,0,0.6156176090240478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,4,2,128,1,fp8,fp8,0,0.6125216007232666
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,4,4,128,1,float16,float16,0,0.3605839967727661
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,4,4,128,1,float16,fp8,0,0.3329792022705078
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,4,2,128,1,float16,float16,0,0.0623088002204895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,4,4,128,1,fp8,fp8,0,0.33131520748138427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,4,1,128,1,float16,fp8,0,0.1233680009841919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,4,1,128,1,fp8,fp8,0,0.3189903974533081
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,4,2,128,1,float16,float16,0,0.33428800106048584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,4,2,128,1,float16,fp8,0,0.31808159351348875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,4,4,128,1,float16,float16,0,0.19128960371017456
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,4,2,128,1,fp8,fp8,0,0.31908159255981444
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,4,1,128,1,float16,fp8,0,0.6131296157836914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,4,4,128,1,float16,fp8,0,0.17547680139541627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,4,4,128,1,fp8,fp8,0,0.17580480575561525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,4,1,128,1,float16,float16,0,0.1756592035293579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,4,1,128,1,float16,fp8,0,0.16877280473709105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,4,2,128,1,float16,float16,0,0.17954879999160767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,4,2,128,1,float16,fp8,0,0.16962560415267944
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,4,2,128,1,fp8,fp8,0,0.16957440376281738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,4,4,128,1,float16,float16,0,0.10494240522384643
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,4,4,128,1,float16,fp8,0,0.0959280014038086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,4,4,128,1,fp8,fp8,0,0.09615200161933898
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,4,1,128,1,float16,float16,0,0.0943008005619049
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,4,4,128,1,float16,float16,0,0.5853472232818604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,4,1,128,1,fp8,fp8,0,0.09221280217170716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,4,2,128,1,float16,float16,0,0.09879999756813049
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,4,2,128,1,float16,fp8,0,0.09302080273628235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,4,2,128,1,fp8,fp8,0,0.09242399930953979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,4,4,128,1,float16,float16,0,0.05856159925460815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,4,4,128,1,float16,fp8,0,0.05588639974594116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,4,4,128,1,fp8,fp8,0,0.05539360046386719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,4,1,128,1,float16,float16,0,0.054630398750305176
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,4,1,128,1,float16,fp8,0,0.05559520125389099
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,4,1,128,1,fp8,fp8,0,0.05530080199241638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,4,2,128,1,float16,fp8,0,0.05524960160255432
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,4,2,128,1,fp8,fp8,0,0.055017602443695066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,4,4,128,1,float16,float16,0,0.05253919959068298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,4,4,128,1,float16,fp8,0,0.05101760029792786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,4,4,128,1,fp8,fp8,0,0.05116479992866516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,4,1,128,1,float16,float16,0,0.04174880087375641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,4,1,128,1,float16,fp8,0,0.04183200001716614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,4,4,128,1,float16,fp8,0,0.13621920347213745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,4,1,128,1,float16,float16,0,0.3270256042480469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,4,1,128,1,fp8,fp8,0,0.04145280122756958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,4,2,128,1,float16,float16,0,0.04846239984035492
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,4,1,128,1,float16,fp8,0,0.31579999923706054
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,4,2,128,1,fp8,fp8,0,0.04860160052776337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,4,1,128,1,float16,float16,0,0.6299007892608642
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,4,1,128,1,fp8,fp8,0,0.6139743804931641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,4,2,128,1,float16,float16,0,0.6548031806945801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,4,1,128,1,fp8,fp8,0,0.16785119771957396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,4,4,128,1,float16,float16,0,0.37223200798034667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,4,2,128,1,float16,fp8,0,0.6206592082977295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,4,4,128,1,float16,fp8,0,0.3417664051055908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,4,2,128,1,fp8,fp8,0,0.6223008155822753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,4,1,128,1,float16,fp8,0,0.09247999787330627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,4,4,128,1,fp8,fp8,0,0.33960959911346433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,4,1,128,1,float16,float16,0,0.32826240062713624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,4,1,128,1,float16,fp8,0,0.31768639087677003
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,4,1,128,1,fp8,fp8,0,0.3186367988586426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,4,2,128,1,float16,float16,0,0.3397552013397217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,4,2,128,1,float16,fp8,0,0.3198767900466919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,4,4,128,1,float16,float16,0,0.19838080406188965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,4,4,128,1,float16,fp8,0,0.18264319896697997
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,4,2,128,1,float16,float16,0,0.05480480194091797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,4,4,128,1,fp8,fp8,0,0.18242720365524293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,4,1,128,1,float16,float16,0,0.1761952042579651
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,4,1,128,1,float16,fp8,0,0.1699120044708252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,4,1,128,1,fp8,fp8,0,0.17021119594573975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,4,2,128,1,float16,float16,0,0.18179999589920043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,4,2,128,1,float16,fp8,0,0.1720703959465027
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,4,4,128,1,float16,float16,0,0.11060960292816162
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,4,4,128,1,fp8,fp8,0,0.10320960283279419
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,4,1,128,1,float16,float16,0,0.09854239821434022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,4,1,128,1,float16,fp8,0,0.09407520294189453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,4,1,128,1,fp8,fp8,0,0.09369120001792908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,4,2,128,1,float16,fp8,0,0.04860639870166779
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,4,2,128,1,float16,float16,0,0.10162080526351928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,4,2,128,1,float16,fp8,0,0.09578239917755127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,4,2,128,1,fp8,fp8,0,0.09561600089073181
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,4,4,128,1,float16,float16,0,0.06619840264320373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,4,4,128,1,float16,fp8,0,0.0593936026096344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,4,4,128,1,fp8,fp8,0,0.05931680202484131
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,4,1,128,1,float16,float16,0,0.05849599838256836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,4,1,128,1,float16,fp8,0,0.05850719809532166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,4,1,128,1,fp8,fp8,0,0.05832800269126892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,4,2,128,1,float16,fp8,0,0.05850239992141724
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,4,1,128,1,float16,fp8,0,0.6186751842498779
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,4,2,128,1,fp8,fp8,0,0.058671998977661136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,4,4,128,1,float16,float16,0,0.04156480133533478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,4,4,128,1,float16,fp8,0,0.0404992014169693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,4,4,128,1,fp8,fp8,0,0.040622401237487796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,4,1,128,1,float16,fp8,0,0.040347200632095334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,4,1,128,1,fp8,fp8,0,0.04012320041656494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,4,2,128,1,float16,float16,0,0.0403439998626709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,4,2,128,1,fp8,fp8,0,0.04020160138607025
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,4,4,128,1,float16,float16,0,0.032393598556518556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,4,4,128,1,float16,fp8,0,0.032583999633789065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,4,4,128,1,fp8,fp8,0,0.03280639946460724
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,4,1,128,1,float16,float16,0,0.03203679919242859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,4,1,128,1,float16,fp8,0,0.03290719985961914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,4,1,128,1,fp8,fp8,0,0.03303839862346649
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,4,2,128,1,float16,float16,0,0.031857600808143614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,4,2,128,1,fp8,fp8,0,0.32256479263305665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,4,2,128,1,fp8,fp8,0,0.03269279897212982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,4,2,128,1,fp8,fp8,0,0.17136800289154053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,4,1,128,1,float16,float16,0,0.4066927909851074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,4,1,128,1,float16,fp8,0,0.3936271905899048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,4,1,128,1,fp8,fp8,0,0.39852960109710694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,4,2,128,1,float16,float16,0,0.059297597408294676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,4,1,128,1,float16,float16,0,0.04001759886741638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,4,2,128,1,float16,fp8,0,0.04033919870853424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,4,4,128,1,float16,float16,0,0.24556639194488525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,4,4,128,1,float16,fp8,0,0.22599680423736573
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,4,4,128,1,fp8,fp8,0,0.22657759189605714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,4,1,128,1,float16,float16,0,0.2144223928451538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,4,1,128,1,float16,fp8,0,0.20821759700775147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,4,1,128,1,fp8,fp8,0,0.20772318840026854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,4,2,128,1,float16,fp8,0,0.03243519961833954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,4,2,128,1,float16,float16,0,0.22215840816497803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,4,4,128,1,float16,float16,0,0.1348207950592041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,4,2,128,1,float16,fp8,0,0.21138401031494142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,4,2,128,1,fp8,fp8,0,0.21200480461120605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,4,4,128,1,float16,fp8,0,0.1234015941619873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,4,4,128,1,float16,fp8,0,0.10345760583877564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,4,4,128,1,fp8,fp8,0,0.12398719787597656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,4,1,128,1,float16,fp8,0,0.11439360380172729
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,4,2,128,1,float16,float16,0,0.1222607970237732
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,4,2,128,1,float16,fp8,0,0.11653120517730713
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,4,2,128,1,fp8,fp8,0,0.11635359525680541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,4,4,128,1,float16,fp8,0,0.07269600033760071
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,4,4,128,1,fp8,fp8,0,0.07251999974250793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,4,1,128,1,float16,float16,0,0.06688479781150818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,4,1,128,1,float16,fp8,0,0.06569280028343201
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,4,1,128,1,fp8,fp8,0,0.06580640077590942
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,4,2,128,1,float16,float16,0,0.07095999717712402
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,4,2,128,1,float16,fp8,0,0.06590399742126465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,4,2,128,1,float16,float16,0,0.418943977355957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,4,4,128,1,float16,float16,0,0.04521439969539642
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,4,4,128,1,fp8,fp8,0,0.04312959909439087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,4,2,128,1,fp8,fp8,0,0.40333118438720705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,4,1,128,1,float16,float16,0,0.041417598724365234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,4,1,128,1,float16,fp8,0,0.04213919937610626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,4,1,128,1,fp8,fp8,0,0.04233280122280121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,4,2,128,1,float16,float16,0,0.04123679995536804
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,4,2,128,1,float16,fp8,0,0.04238399863243103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,4,2,128,1,fp8,fp8,0,0.0424560010433197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,4,4,128,1,float16,float16,0,0.04012480080127716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,4,4,128,1,float16,fp8,0,0.03962399959564209
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,4,4,128,1,fp8,fp8,0,0.03987999856472015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,4,1,128,1,float16,float16,0,0.037145599722862244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,4,1,128,1,float16,fp8,0,0.03774400055408478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,4,1,128,1,fp8,fp8,0,0.037745600938796996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,4,2,128,1,float16,float16,0,0.03858239948749542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,4,2,128,1,float16,fp8,0,0.039468801021575926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,4,2,128,1,fp8,fp8,0,0.03975679874420166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,4,4,128,1,float16,float16,0,0.02614719867706299
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,4,4,128,1,float16,fp8,0,0.02651839852333069
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,4,4,128,1,fp8,fp8,0,0.026345598697662353
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,4,1,128,1,float16,float16,0,0.024766400456428528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,4,1,128,1,float16,fp8,0,0.026137599349021913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,4,1,128,1,fp8,fp8,0,0.025892800092697142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,4,2,128,1,float16,float16,0,0.02491839975118637
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,4,2,128,1,float16,fp8,0,0.02612000107765198
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,4,2,128,1,fp8,fp8,0,0.02620159983634949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,4,1,128,1,float16,float16,0,0.11709920167922974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,4,1,128,1,fp8,fp8,0,0.1147536039352417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,4,4,128,1,float16,float16,0,0.07719359993934631
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,4,1,128,1,float16,float16,0,0.431171178817749
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,4,1,128,1,float16,fp8,0,0.4292816162109375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,4,1,128,1,fp8,fp8,0,0.4293168067932129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,4,2,128,1,float16,float16,0,0.4508848190307617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,4,2,128,1,float16,fp8,0,0.43828320503234863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,4,4,128,1,float16,float16,0,0.2688911914825439
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,4,4,128,1,float16,fp8,0,0.24996960163116455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,4,4,128,1,fp8,fp8,0,0.24953761100769042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,4,2,128,1,float16,fp8,0,0.40572161674499513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,4,1,128,1,float16,float16,0,0.2261568069458008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,4,1,128,1,float16,fp8,0,0.2248176097869873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,4,1,128,1,fp8,fp8,0,0.22418880462646484
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,4,2,128,1,float16,fp8,0,0.22920639514923097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,4,4,128,1,float16,float16,0,0.14399839639663697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,4,2,128,1,fp8,fp8,0,0.22920479774475097
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,4,4,128,1,float16,fp8,0,0.134225594997406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,4,4,128,1,fp8,fp8,0,0.13427200317382812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,4,1,128,1,float16,float16,0,0.12187999486923218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,4,1,128,1,float16,fp8,0,0.12094399929046631
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,4,1,128,1,fp8,fp8,0,0.1211135983467102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,4,2,128,1,float16,float16,0,0.12786879539489746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,4,2,128,1,float16,fp8,0,0.12348639965057373
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,4,4,128,1,float16,float16,0,0.08110719919204712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,4,4,128,1,float16,fp8,0,0.07667679786682129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,4,4,128,1,fp8,fp8,0,0.07614240050315857
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,4,1,128,1,float16,float16,0,0.06938239932060242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,4,1,128,1,float16,fp8,0,0.06771039962768555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,4,1,128,1,fp8,fp8,0,0.0673695981502533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,4,2,128,1,float16,float16,0,0.07261919975280762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,4,2,128,1,float16,fp8,0,0.06993280053138733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,4,2,128,1,fp8,fp8,0,0.0695743978023529
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,4,4,128,1,float16,float16,0,0.05009120106697083
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,4,4,128,1,float16,fp8,0,0.04416640102863312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,4,4,128,1,fp8,fp8,0,0.04360319972038269
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,4,1,128,1,float16,float16,0,0.042531201243400575
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,4,1,128,1,float16,fp8,0,0.04302079975605011
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,4,1,128,1,fp8,fp8,0,0.04290719926357269
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,4,2,128,1,float16,float16,0,0.04319519996643066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,4,2,128,1,float16,fp8,0,0.04303680062294006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,4,2,128,1,fp8,fp8,0,0.04320319890975952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,4,2,128,1,fp8,fp8,0,0.06620479822158813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,4,2,128,1,fp8,fp8,0,0.43804001808166504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,4,4,128,1,float16,float16,0,0.02876960039138794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,4,4,128,1,fp8,fp8,0,0.028718400001525878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,4,1,128,1,float16,fp8,0,0.028254398703575136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,4,1,128,1,float16,float16,0,0.02747359871864319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,4,2,128,1,float16,float16,0,0.027783998847007753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,4,2,128,1,float16,fp8,0,0.028436800837516783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,4,2,128,1,fp8,fp8,0,0.028262400627136232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,4,4,128,1,float16,float16,0,0.026281601190567015
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,4,4,128,1,float16,fp8,0,0.026939201354980468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,4,4,128,1,fp8,fp8,0,0.027193599939346315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,4,2,128,1,float16,float16,0,0.23728480339050292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,4,1,128,1,float16,float16,0,0.025543999671936036
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,4,1,128,1,float16,fp8,0,0.0273360013961792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,4,2,128,1,float16,float16,0,0.02549920082092285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,4,2,128,1,float16,fp8,0,0.02712000012397766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,4,2,128,1,fp8,fp8,0,0.02687999904155731
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,4,4,128,1,float16,fp8,0,0.02120479941368103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,4,4,128,1,fp8,fp8,0,0.02125920057296753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,4,1,128,1,float16,float16,0,0.021121600270271303
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,4,1,128,1,float16,fp8,0,0.020561599731445314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,4,2,128,1,fp8,fp8,0,0.12371840476989746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,4,1,128,1,fp8,fp8,0,0.02048799991607666
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,4,2,128,1,float16,fp8,0,0.020729599893093108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,4,2,128,1,float16,float16,0,0.021198399364948273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,4,2,128,1,fp8,fp8,0,0.0207056000828743
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,4,1,128,1,float16,float16,0,0.27901759147644045
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,4,1,128,1,float16,fp8,0,0.2887279987335205
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,4,1,128,1,fp8,fp8,0,0.2884864091873169
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,4,2,128,1,float16,float16,0,0.3024336099624634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,4,2,128,1,float16,fp8,0,0.2988111972808838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,4,2,128,1,fp8,fp8,0,0.2993824005126953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,4,4,128,1,float16,float16,0,0.19224799871444703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,4,4,128,1,float16,fp8,0,0.1782464027404785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,4,4,128,1,float16,fp8,0,0.042817598581314086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,4,1,128,1,fp8,fp8,0,0.028377598524093627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,4,1,128,1,float16,fp8,0,0.15161759853363038
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,4,1,128,1,fp8,fp8,0,0.1509119987487793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,4,2,128,1,float16,float16,0,0.1596127986907959
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,4,2,128,1,float16,fp8,0,0.15729279518127443
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,4,4,128,1,float16,float16,0,0.1036911964416504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,4,4,128,1,float16,fp8,0,0.09728320240974427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,4,1,128,1,fp8,fp8,0,0.027296000719070436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,4,1,128,1,float16,float16,0,0.08148639798164367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,4,4,128,1,float16,float16,0,0.021495999395847322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,4,1,128,1,float16,fp8,0,0.08384960293769836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,4,1,128,1,fp8,fp8,0,0.08367679715156555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,4,2,128,1,float16,float16,0,0.08751680254936219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,4,2,128,1,float16,fp8,0,0.08651999831199646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,4,2,128,1,fp8,fp8,0,0.0865119993686676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,4,4,128,1,float16,float16,0,0.05993279814720154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,4,4,128,1,float16,fp8,0,0.056492799520492555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,4,4,128,1,fp8,fp8,0,0.05602399706840515
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,4,1,128,1,float16,float16,0,0.04775680005550385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,4,1,128,1,float16,fp8,0,0.04668320119380951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,4,1,128,1,fp8,fp8,0,0.04677599966526032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,4,2,128,1,float16,float16,0,0.051502400636672975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,4,2,128,1,float16,fp8,0,0.04951840043067932
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,4,2,128,1,fp8,fp8,0,0.049265599250793456
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,4,4,128,1,float16,float16,0,0.03593760132789612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,4,4,128,1,fp8,fp8,0,0.03182240128517151
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,4,1,128,1,float16,float16,0,0.028638398647308348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,4,1,128,1,float16,fp8,0,0.029721599817276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,4,1,128,1,fp8,fp8,0,0.03003840148448944
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,4,2,128,1,float16,float16,0,0.028787198662757873
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,4,2,128,1,float16,fp8,0,0.030191999673843384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,4,2,128,1,fp8,fp8,0,0.030372801423072814
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,4,4,128,1,float16,float16,0,0.019926400482654573
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,4,4,128,1,float16,fp8,0,0.020414400100708007
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,4,4,128,1,fp8,fp8,0,0.020388799905776977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,4,1,128,1,float16,float16,0,0.018603199720382692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,4,1,128,1,float16,fp8,0,0.02006399929523468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,4,1,128,1,fp8,fp8,0,0.020032000541687012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,4,2,128,1,float16,float16,0,0.01875839978456497
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,4,4,128,1,float16,fp8,0,0.02887519896030426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,4,2,128,1,float16,fp8,0,0.020193600654602052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,4,2,128,1,fp8,fp8,0,0.020241600275039674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,4,4,128,1,float16,float16,0,0.018483200669288637
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,4,1,128,1,float16,float16,0,0.14814079999923707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,4,4,128,1,float16,fp8,0,0.019675199687480927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,4,4,128,1,fp8,fp8,0,0.019710400700569154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,4,1,128,1,float16,float16,0,0.01763039976358414
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,4,1,128,1,float16,fp8,0,0.019166399538517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,4,1,128,1,fp8,fp8,0,0.019091199338436126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,4,2,128,1,float16,float16,0,0.01781439930200577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,4,2,128,1,float16,fp8,0,0.019100800156593323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,4,4,128,1,float16,float16,0,0.017771199345588684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,4,2,128,1,fp8,fp8,0,0.15656479597091674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,4,1,128,1,float16,float16,0,0.01748320013284683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,4,4,128,1,fp8,fp8,0,0.09689279794692993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,4,1,128,1,float16,fp8,0,0.018900799751281738
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,4,1,128,1,fp8,fp8,0,0.018665599822998046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,4,2,128,1,float16,float16,0,0.017347200214862822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,4,2,128,1,float16,fp8,0,0.018705600500106813
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,4,2,128,1,fp8,fp8,0,0.01902720034122467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,4,4,128,1,float16,float16,0,0.01684480011463165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,4,4,128,1,float16,fp8,0,0.019313600659370423
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,4,4,128,1,fp8,fp8,0,0.01882079988718033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,4,1,128,1,float16,float16,0,0.01672320067882538
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,4,1,128,1,float16,fp8,0,0.01857919991016388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,4,1,128,1,fp8,fp8,0,0.018614399433135986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,4,2,128,1,float16,float16,0,0.016769599914550782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,4,2,128,1,float16,fp8,0,0.018753600120544434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,4,2,128,1,fp8,fp8,0,0.01886080056428909
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,4,4,128,1,float16,fp8,0,0.03196159899234772
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,4,1,128,1,float16,fp8,0,0.13105440139770508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,4,1,128,1,fp8,fp8,0,0.13084479570388793
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,4,2,128,1,float16,float16,0,0.13729599714279175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,4,2,128,1,float16,fp8,0,0.13643360137939453
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,4,2,128,1,fp8,fp8,0,0.13578879833221436
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,4,4,128,1,float16,float16,0,0.09426400065422058
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,4,4,128,1,float16,fp8,0,0.08275200128555298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,4,4,128,1,fp8,fp8,0,0.0827679991722107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,4,1,128,1,float16,fp8,0,0.0721231997013092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,4,1,128,1,fp8,fp8,0,0.0720624029636383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,4,4,128,1,fp8,fp8,0,0.17836960554122924
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,4,2,128,1,float16,fp8,0,0.07442560195922851
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,4,2,128,1,fp8,fp8,0,0.07466239929199218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,4,2,128,1,fp8,fp8,0,0.01908160001039505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,4,4,128,1,float16,float16,0,0.054148799180984496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,4,4,128,1,float16,fp8,0,0.04702880084514618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,4,4,128,1,fp8,fp8,0,0.0472815990447998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,4,1,128,1,float16,float16,0,0.041377601027488706
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,4,1,128,1,float16,fp8,0,0.03993920087814331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,4,1,128,1,fp8,fp8,0,0.04017919898033142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,4,2,128,1,float16,float16,0,0.04475359916687012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,4,2,128,1,float16,fp8,0,0.041838398575782774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,4,2,128,1,fp8,fp8,0,0.04189600050449371
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,4,4,128,1,float16,float16,0,0.03148640096187592
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,4,4,128,1,float16,fp8,0,0.025726398825645445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,4,4,128,1,fp8,fp8,0,0.02579039931297302
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,4,1,128,1,float16,float16,0,0.023470400273799895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,4,1,128,1,float16,fp8,0,0.025283199548721314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,4,1,128,1,fp8,fp8,0,0.02512960135936737
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,4,2,128,1,float16,fp8,0,0.025390401482582092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,4,1,128,1,float16,float16,0,0.12453279495239258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,4,2,128,1,fp8,fp8,0,0.02534080147743225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,4,4,128,1,float16,float16,0,0.017177599668502807
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,4,4,128,1,float16,fp8,0,0.01669439971446991
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,4,4,128,1,fp8,fp8,0,0.016817599534988403
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,4,1,128,1,float16,float16,0,0.015489600598812103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,4,1,128,1,float16,fp8,0,0.016340799629688263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,4,2,128,1,float16,float16,0,0.015779200196266174
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,4,2,128,1,float16,fp8,0,0.016334399580955505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,4,2,128,1,fp8,fp8,0,0.016264000535011293
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,4,4,128,1,float16,float16,0,0.015340800583362579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,4,1,128,1,float16,float16,0,0.0699504017829895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,4,4,128,1,float16,fp8,0,0.015934400260448456
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,4,4,128,1,fp8,fp8,0,0.015727999806404113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,4,1,128,1,float16,float16,0,0.014596800506114959
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,4,1,128,1,float16,fp8,0,0.015377600491046906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,4,2,128,1,float16,float16,0,0.01486240029335022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,4,2,128,1,float16,fp8,0,0.01555519998073578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,4,2,128,1,float16,float16,0,0.07615039944648742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,4,2,128,1,fp8,fp8,0,0.015646399557590486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,4,4,128,1,float16,float16,0,0.014777599275112152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,4,4,128,1,fp8,fp8,0,0.01539520025253296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,4,1,128,1,float16,float16,0,0.014254400134086609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,4,1,128,1,float16,fp8,0,0.01486240029335022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,4,1,128,1,fp8,fp8,0,0.01499360054731369
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,4,2,128,1,float16,float16,0,0.014169600605964661
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,4,4,128,1,float16,fp8,0,0.01929280012845993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,4,2,128,1,fp8,fp8,0,0.01523679941892624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,4,4,128,1,float16,float16,0,0.013967999815940857
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,4,4,128,1,fp8,fp8,0,0.015067200362682342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,4,2,128,1,float16,float16,0,0.023835200071334838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,4,1,128,1,float16,float16,0,0.013817599415779114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,4,1,128,1,float16,fp8,0,0.014800000190734863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,4,1,128,1,fp8,fp8,0,0.014881600439548493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,4,2,128,1,float16,float16,0,0.013939200341701508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,4,1,128,1,fp8,fp8,0,0.0162992000579834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,4,2,128,1,fp8,fp8,0,0.015038399398326874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,4,4,128,1,float16,float16,0,0.013886399567127228
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,4,4,128,1,float16,fp8,0,0.01494240015745163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,4,4,128,1,fp8,fp8,0,0.014772799611091614
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,4,1,128,1,float16,float16,0,0.013631999492645264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,4,1,128,1,float16,fp8,0,0.014753599464893342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,4,1,128,1,fp8,fp8,0,0.015358400344848634
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,4,1,128,1,fp8,fp8,0,0.014803199470043183
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,4,2,128,1,float16,float16,0,0.013628800213336945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,4,2,128,1,float16,fp8,0,0.014932799339294433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,4,2,128,1,fp8,fp8,0,0.014681600034236908
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,4,1,128,1,float16,float16,0,0.0670415997505188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,4,4,128,1,float16,fp8,0,0.015471999347209931
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,4,1,128,1,fp8,fp8,0,0.06866559982299805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,4,2,128,1,float16,float16,0,0.0743503987789154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,4,2,128,1,float16,fp8,0,0.015222400426864624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,4,4,128,1,fp8,fp8,0,0.019223999977111817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,4,4,128,1,float16,fp8,0,0.014948800206184387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,4,4,128,1,float16,fp8,0,0.043433600664138795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,4,4,128,1,fp8,fp8,0,0.04299359917640686
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,4,1,128,1,float16,float16,0,0.03845919966697693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,4,1,128,1,float16,fp8,0,0.036353600025177
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,4,1,128,1,fp8,fp8,0,0.035964798927307126
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,4,2,128,1,float16,float16,0,0.04255039989948273
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,4,2,128,1,fp8,fp8,0,0.037544000148773196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,4,4,128,1,float16,float16,0,0.02903040051460266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,4,4,128,1,float16,fp8,0,0.02186720073223114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,4,4,128,1,fp8,fp8,0,0.021910400688648225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,4,1,128,1,float16,float16,0,0.020580799877643587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,4,1,128,1,float16,fp8,0,0.021873599290847777
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,4,1,128,1,fp8,fp8,0,0.02194399982690811
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,4,2,128,1,float16,float16,0,0.02101919949054718
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,4,2,128,1,float16,fp8,0,0.02197919934988022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,4,2,128,1,fp8,fp8,0,0.02203039973974228
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,4,4,128,1,float16,float16,0,0.016276800632476808
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,4,4,128,1,float16,fp8,0,0.015508800745010376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,4,4,128,1,fp8,fp8,0,0.015488000214099884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,4,1,128,1,float16,float16,0,0.014796799421310425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,4,1,128,1,float16,fp8,0,0.015323199331760406
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,4,1,128,1,fp8,fp8,0,0.015217599272727967
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,4,2,128,1,float16,float16,0,0.014798399806022645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,4,2,128,1,float16,fp8,0,0.015248000621795654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,4,2,128,1,fp8,fp8,0,0.015107199549674988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,4,4,128,1,float16,float16,0,0.014180800318717957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,4,4,128,1,float16,fp8,0,0.013819199800491334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,4,4,128,1,fp8,fp8,0,0.013920000195503235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,4,1,128,1,float16,float16,0,0.013052800297737121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,4,1,128,1,float16,fp8,0,0.014027200639247894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,4,1,128,1,fp8,fp8,0,0.013670399785041809
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,4,2,128,1,float16,float16,0,0.013716800510883332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,4,2,128,1,float16,fp8,0,0.0135903999209404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,4,2,128,1,fp8,fp8,0,0.013911999762058258
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,4,4,128,1,float16,float16,0,0.01308320015668869
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,4,4,128,1,float16,fp8,0,0.013531200587749481
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,4,4,128,1,fp8,fp8,0,0.013400000333786011
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,4,1,128,1,float16,fp8,0,0.013460800051689148
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,4,1,128,1,fp8,fp8,0,0.013246400654315949
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,4,2,128,1,fp8,fp8,0,0.07042719721794129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,4,4,128,1,float16,float16,0,0.05237759947776795
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,4,2,128,1,float16,fp8,0,0.013575999438762665
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,4,4,128,1,float16,float16,0,0.012539200484752655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,4,4,128,1,float16,fp8,0,0.013121600449085235
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,4,4,128,1,fp8,fp8,0,0.012984000146389008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,4,1,128,1,float16,float16,0,0.012302400171756744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,4,2,128,1,float16,fp8,0,0.03758240044116974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,4,1,128,1,float16,fp8,0,0.013128000497817992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,4,1,128,1,fp8,fp8,0,0.012804800271987915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,4,2,128,1,float16,fp8,0,0.013118399679660797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,4,2,128,1,fp8,fp8,0,0.012928000092506409
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,4,4,128,1,float16,float16,0,0.012167999893426895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,4,4,128,1,float16,fp8,0,0.012628799676895142
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,4,4,128,1,fp8,fp8,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,4,1,128,1,float16,float16,0,0.011937599629163742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,4,1,128,1,float16,fp8,0,0.01263200044631958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,4,1,128,1,fp8,fp8,0,0.012755200266838074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,4,2,128,1,float16,fp8,0,0.015056000649929046
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,4,2,128,1,float16,float16,0,0.012211199849843979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,4,2,128,1,float16,fp8,0,0.012873600423336028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,4,2,128,1,fp8,fp8,0,0.012774400413036346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,4,4,128,1,float16,float16,0,0.01215839982032776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,4,4,128,1,float16,fp8,0,0.012697599828243256
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,4,4,128,1,fp8,fp8,0,0.01279039978981018
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,4,1,128,1,float16,float16,0,0.012067200243473053
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,4,1,128,1,float16,fp8,0,0.012889599800109864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,4,1,128,1,fp8,fp8,0,0.012689599394798278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,4,2,128,1,float16,float16,0,0.012083200365304947
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,4,2,128,1,float16,fp8,0,0.012571200728416443
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,4,2,128,1,fp8,fp8,0,0.012683199346065521
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,4,1,128,1,float16,float16,0,0.03917919993400574
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,4,1,128,1,float16,fp8,0,0.03797760009765625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,4,1,128,1,fp8,fp8,0,0.03789600133895874
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,4,2,128,1,float16,float16,0,0.04291360080242157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,4,2,128,1,float16,fp8,0,0.03913280069828033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,4,2,128,1,fp8,fp8,0,0.03941439986228943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,4,4,128,1,float16,float16,0,0.03757599890232086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,4,1,128,1,float16,fp8,0,0.06836159825325012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,4,4,128,1,float16,fp8,0,0.03258079886436462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,4,1,128,1,float16,float16,0,0.01263359934091568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,4,4,128,1,fp8,fp8,0,0.03176479935646057
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,4,1,128,1,float16,float16,0,0.021991999447345735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,4,1,128,1,float16,fp8,0,0.023443199694156647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,4,1,128,1,fp8,fp8,0,0.02277279943227768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,4,2,128,1,float16,float16,0,0.013040000200271606
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,4,4,128,1,float16,float16,0,0.020057600736618043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,4,4,128,1,float16,fp8,0,0.020411199331283568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,4,2,128,1,fp8,fp8,0,0.013415999710559845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,4,4,128,1,fp8,fp8,0,0.02037599980831146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,4,1,128,1,float16,float16,0,0.015094399452209473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,4,1,128,1,float16,fp8,0,0.015931199491024017
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,4,1,128,1,fp8,fp8,0,0.015854400396347047
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,4,2,128,1,float16,float16,0,0.015624000132083893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,4,2,128,1,float16,fp8,0,0.015855999290943147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,4,2,128,1,fp8,fp8,0,0.015883199870586395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,4,2,128,1,float16,float16,0,0.012478400021791458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,4,4,128,1,float16,fp8,0,0.014150400459766389
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,4,4,128,1,fp8,fp8,0,0.014343999326229095
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,4,1,128,1,float16,fp8,0,0.01422400027513504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,4,1,128,1,fp8,fp8,0,0.014059199392795563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,4,2,128,1,float16,float16,0,0.013593600690364837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,4,2,128,1,float16,fp8,0,0.014233599603176116
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,4,2,128,1,fp8,fp8,0,0.014004799723625182
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,4,4,128,1,float16,float16,0,0.013009600341320038
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,4,4,128,1,float16,fp8,0,0.013571199774742127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,4,4,128,1,fp8,fp8,0,0.0135903999209404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,4,1,128,1,float16,float16,0,0.013118399679660797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,4,1,128,1,float16,fp8,0,0.013340799510478974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,4,1,128,1,fp8,fp8,0,0.013819199800491334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,4,2,128,1,float16,float16,0,0.01271200031042099
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,4,2,128,1,float16,fp8,0,0.01377439945936203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,4,2,128,1,fp8,fp8,0,0.013593600690364837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,4,4,128,1,float16,float16,0,0.012398400157690049
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,4,4,128,1,float16,fp8,0,0.01292479932308197
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,4,4,128,1,fp8,fp8,0,0.013364799320697784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,4,1,128,1,float16,float16,0,0.012515200674533844
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,4,1,128,1,float16,fp8,0,0.013260799646377563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,4,1,128,1,fp8,fp8,0,0.013275200128555298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,4,2,128,1,float16,fp8,0,0.06995840072631836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,4,2,128,1,float16,float16,0,0.012465599924325943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,4,2,128,1,float16,fp8,0,0.02335679978132248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,4,2,128,1,float16,fp8,0,0.013174399733543396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,4,4,128,1,float16,float16,0,0.012091200053691863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,4,4,128,1,float16,fp8,0,0.01271039992570877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,4,4,128,1,fp8,fp8,0,0.012870399653911591
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,4,1,128,1,float16,float16,0,0.012088000029325485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,4,1,128,1,float16,fp8,0,0.012775999307632447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,4,2,128,1,float16,float16,0,0.012206400185823441
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,4,2,128,1,float16,fp8,0,0.012839999794960023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,4,2,128,1,fp8,fp8,0,0.01281919926404953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,4,4,128,1,float16,float16,0,0.014134399592876434
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,4,4,128,1,float16,fp8,0,0.0126351997256279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,4,1,128,1,float16,float16,0,0.013302400708198547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,4,4,128,1,fp8,fp8,0,0.01252480000257492
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,4,1,128,1,float16,float16,0,0.01207199990749359
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,4,1,128,1,fp8,fp8,0,0.012539200484752655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,4,2,128,1,float16,float16,0,0.011937599629163742
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,4,2,128,1,float16,fp8,0,0.012643200159072877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,4,2,128,1,fp8,fp8,0,0.01260959953069687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,4,4,128,1,float16,float16,0,0.011716800183057785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,4,4,128,1,float16,fp8,0,0.012494400143623352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,4,4,128,1,fp8,fp8,0,0.012398400157690049
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,4,1,128,1,float16,float16,0,0.011689600348472596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,4,1,128,1,float16,fp8,0,0.012404800206422806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,4,1,128,1,fp8,fp8,0,0.012412799894809723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,4,2,128,1,float16,float16,0,0.01186240017414093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,4,2,128,1,float16,fp8,0,0.01234399974346161
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,4,2,128,1,fp8,fp8,0,0.012638400495052337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,4,1,128,1,float16,float16,0,0.023588800430297853
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,4,1,128,1,float16,fp8,0,0.026044800877571106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,4,1,128,1,fp8,fp8,0,0.025515198707580566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,4,2,128,1,float16,float16,0,0.03165760040283203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,4,2,128,1,float16,fp8,0,0.03394719958305359
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,4,2,128,1,fp8,fp8,0,0.03464480042457581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,4,4,128,1,float16,float16,0,0.028502398729324342
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,4,4,128,1,float16,fp8,0,0.03057120144367218
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,4,4,128,1,fp8,fp8,0,0.03054560124874115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,4,1,128,1,float16,float16,0,0.016171200573444365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,4,1,128,1,float16,fp8,0,0.01741439998149872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,4,1,128,1,fp8,fp8,0,0.016889600455760954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,4,2,128,1,float16,float16,0,0.019852800667285918
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,4,2,128,1,float16,fp8,0,0.02126239985227585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,4,2,128,1,fp8,fp8,0,0.021371200680732727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,4,2,128,1,fp8,fp8,0,0.013262400031089782
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,4,4,128,1,float16,fp8,0,0.019219200313091277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,4,4,128,1,fp8,fp8,0,0.01936960071325302
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,4,1,128,1,float16,fp8,0,0.01478559970855713
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,4,1,128,1,fp8,fp8,0,0.014764800667762756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,4,2,128,1,float16,float16,0,0.014000000059604644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,4,2,128,1,float16,fp8,0,0.014603200554847717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,4,4,128,1,float16,float16,0,0.011919999867677689
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,4,2,128,1,fp8,fp8,0,0.014793600142002105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,4,4,128,1,float16,float16,0,0.013275200128555298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,4,4,128,1,float16,fp8,0,0.01414559930562973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,4,4,128,1,fp8,fp8,0,0.013865600526332855
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,4,1,128,1,float16,float16,0,0.01345440000295639
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,4,1,128,1,float16,fp8,0,0.01265919953584671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,4,1,128,1,fp8,fp8,0,0.01388320028781891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,4,2,128,1,float16,float16,0,0.013068799674510957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,4,2,128,1,float16,fp8,0,0.013803200423717498
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,4,2,128,1,fp8,fp8,0,0.013657599687576294
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,4,4,128,1,float16,float16,0,0.012415999919176102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,4,4,128,1,float16,fp8,0,0.013184000551700593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,4,4,128,1,fp8,fp8,0,0.013158400356769562
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,4,1,128,1,float16,float16,0,0.01231039986014366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,4,1,128,1,float16,fp8,0,0.013132800161838532
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,4,1,128,1,fp8,fp8,0,0.013327999413013459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,4,2,128,1,float16,float16,0,0.01242239996790886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,4,2,128,1,float16,fp8,0,0.013264000415802002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,4,2,128,1,fp8,fp8,0,0.013220800459384919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,4,4,128,1,float16,float16,0,0.012534399330615998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,4,4,128,1,float16,fp8,0,0.013147200644016265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,4,4,128,1,fp8,fp8,0,0.013048000633716583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,4,1,128,1,float16,float16,0,0.012244799733161926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,4,2,128,1,float16,float16,0,0.02242400050163269
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,4,2,128,1,fp8,fp8,0,0.022888000309467315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,4,1,128,1,fp8,fp8,0,0.012814399600028992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,4,2,128,1,float16,fp8,0,0.012590399384498597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,4,1,128,1,float16,float16,0,0.013734400272369385
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,4,2,128,1,fp8,fp8,0,0.012699200212955475
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,4,1,128,1,fp8,fp8,0,0.012862400710582733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,4,4,128,1,fp8,fp8,0,0.01279039978981018
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,4,1,128,1,float16,float16,0,0.011990399658679962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,4,1,128,1,float16,fp8,0,0.01279519945383072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,4,1,128,1,fp8,fp8,0,0.012801599502563477
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,4,2,128,1,float16,float16,0,0.011727999895811081
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,4,2,128,1,float16,fp8,0,0.012427199631929398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,4,1,128,1,float16,fp8,0,0.013670399785041809
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,4,4,128,1,float16,float16,0,0.011660800129175187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,4,4,128,1,float16,fp8,0,0.012483199685811996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,4,4,128,1,fp8,fp8,0,0.012443199753761292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,4,1,128,1,float16,float16,0,0.011643200367689132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,4,1,128,1,float16,fp8,0,0.012483199685811996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,4,1,128,1,fp8,fp8,0,0.012455999851226807
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,4,2,128,1,float16,float16,0,0.011588799953460693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,4,2,128,1,float16,fp8,0,0.012278400361537933
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,4,2,128,1,fp8,fp8,0,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,4,4,128,1,float16,float16,0,0.011715199798345566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,4,4,128,1,float16,fp8,0,0.01244639977812767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,4,4,128,1,fp8,fp8,0,0.012459199875593185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,4,1,128,1,float16,float16,0,0.011935999989509583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,4,1,128,1,float16,fp8,0,0.01266079992055893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,4,1,128,1,fp8,fp8,0,0.012608000636100769
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,4,2,128,1,float16,float16,0,0.011584000289440155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,4,2,128,1,float16,fp8,0,0.012399999797344208
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,4,2,128,1,fp8,fp8,0,0.012403199821710587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,4,1,128,1,float16,float16,0,0.02191520035266876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,4,1,128,1,float16,fp8,0,0.012928000092506409
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,4,1,128,1,float16,fp8,0,0.02383359968662262
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,4,4,128,1,float16,float16,0,0.01863519996404648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,4,2,128,1,float16,float16,0,0.02940639853477478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,4,2,128,1,float16,float16,0,0.011931200325489045
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,4,2,128,1,fp8,fp8,0,0.03245919942855835
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,4,4,128,1,float16,float16,0,0.027110400795936584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,4,4,128,1,float16,fp8,0,0.02985759973526001
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,4,1,128,1,float16,float16,0,0.015574400126934052
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,4,1,128,1,float16,fp8,0,0.01634880006313324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,4,1,128,1,fp8,fp8,0,0.01653279960155487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,4,2,128,1,float16,float16,0,0.019443200528621675
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,4,2,128,1,float16,fp8,0,0.02099359929561615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,4,2,128,1,fp8,fp8,0,0.012441600114107132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,4,4,128,1,float16,fp8,0,0.01886720061302185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,4,4,128,1,fp8,fp8,0,0.018875199556350707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,4,1,128,1,float16,float16,0,0.013518400490283966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,4,1,128,1,float16,fp8,0,0.014251199364662171
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,4,1,128,1,fp8,fp8,0,0.01446239948272705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,4,2,128,1,float16,float16,0,0.013600000739097595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,4,2,128,1,float16,fp8,0,0.014366400241851807
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,4,2,128,1,fp8,fp8,0,0.014483200013637542
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,4,4,128,1,float16,float16,0,0.01297439932823181
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,4,4,128,1,float16,fp8,0,0.013758400082588195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,4,4,128,1,fp8,fp8,0,0.013724799454212188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,4,1,128,1,float16,float16,0,0.012736000120639801
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,4,1,128,1,float16,fp8,0,0.013558399677276612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,4,1,128,1,fp8,fp8,0,0.01356479972600937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,4,2,128,1,float16,float16,0,0.01266240030527115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,4,2,128,1,float16,fp8,0,0.013516800105571746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,4,2,128,1,fp8,fp8,0,0.013447999954223633
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,4,4,128,1,float16,float16,0,0.012345600128173827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,4,4,128,1,float16,fp8,0,0.013201600313186646
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,4,4,128,1,fp8,fp8,0,0.013254399597644805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,4,1,128,1,fp8,fp8,0,0.023763200640678404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,4,1,128,1,float16,float16,0,0.012289600074291229
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,4,1,128,1,float16,fp8,0,0.013091200590133667
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,4,2,128,1,float16,fp8,0,0.032507199048995974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,4,2,128,1,float16,float16,0,0.01252640038728714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,4,4,128,1,float16,float16,0,0.011537600308656693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,4,2,128,1,float16,fp8,0,0.013236799836158752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,4,4,128,1,float16,float16,0,0.01196959987282753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,4,4,128,1,float16,fp8,0,0.012915199995040894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,4,4,128,1,fp8,fp8,0,0.012723200023174286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,4,1,128,1,float16,float16,0,0.011854399740695954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,4,1,128,1,float16,fp8,0,0.012680000066757202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,4,1,128,1,fp8,fp8,0,0.012615999579429627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,4,2,128,1,float16,float16,0,0.011744000017642975
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,4,2,128,1,float16,fp8,0,0.012617599964141846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,4,2,128,1,fp8,fp8,0,0.012352000176906585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,4,2,128,1,fp8,fp8,0,0.02080480009317398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,4,4,128,1,float16,float16,0,0.017825600504875184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,4,4,128,1,fp8,fp8,0,0.012361600250005721
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,4,1,128,1,float16,float16,0,0.011486399918794632
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,4,1,128,1,float16,fp8,0,0.012303999811410903
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,4,1,128,1,fp8,fp8,0,0.01228799968957901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,4,2,128,1,float16,float16,0,0.011535999923944473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,4,2,128,1,float16,fp8,0,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,4,2,128,1,fp8,fp8,0,0.012347199767827988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,4,4,128,1,float16,float16,0,0.011726400256156922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,4,4,128,1,float16,fp8,0,0.012353599816560746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,4,4,128,1,fp8,fp8,0,0.012292800098657608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,4,1,128,1,float16,float16,0,0.011582399904727935
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,4,1,128,1,float16,fp8,0,0.012459199875593185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,4,1,128,1,fp8,fp8,0,0.012468799948692322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,4,2,128,1,float16,float16,0,0.011716800183057785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,4,2,128,1,float16,fp8,0,0.012382400035858155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,4,2,128,1,fp8,fp8,0,0.012412799894809723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,4,4,128,1,float16,float16,0,0.011580800265073776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,4,4,128,1,float16,fp8,0,0.012328000366687774
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,4,4,128,1,fp8,fp8,0,0.012329600006341934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,4,1,128,1,float16,float16,0,0.01149120032787323
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,4,1,128,1,float16,fp8,0,0.01234399974346161
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,4,1,128,1,fp8,fp8,0,0.012345600128173827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,4,1,128,1,fp8,fp8,0,0.013014400005340576
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,4,2,128,1,float16,float16,0,0.011588799953460693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,4,2,128,1,float16,fp8,0,0.012337599694728852
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,4,4,128,1,float16,fp8,0,0.012492799758911132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,4,2,128,1,fp8,fp8,0,0.013284799456596375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,4,1,128,1,float16,float16,0,0.021326400339603424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,4,1,128,1,float16,fp8,0,0.022915199398994446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,4,1,128,1,fp8,fp8,0,0.023019200563430785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,4,2,128,1,float16,float16,0,0.028302401304244995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,4,2,128,1,float16,fp8,0,0.031641599535942075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,4,2,128,1,fp8,fp8,0,0.03149920105934143
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,4,4,128,1,float16,float16,0,0.025865599513053894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,4,4,128,1,float16,fp8,0,0.02908160090446472
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,4,1,128,1,float16,float16,0,0.0147024005651474
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,4,1,128,1,float16,fp8,0,0.015913599729537965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,4,1,128,1,fp8,fp8,0,0.015967999398708344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,4,2,128,1,float16,float16,0,0.018489600718021394
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,4,2,128,1,float16,fp8,0,0.020206399261951447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,4,2,128,1,fp8,fp8,0,0.020185600221157073
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,4,4,128,1,float16,float16,0,0.01698080003261566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,4,4,128,1,float16,fp8,0,0.018404799699783325
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,4,4,128,1,fp8,fp8,0,0.018484799563884734
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,4,1,128,1,float16,float16,0,0.013038399815559387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,4,1,128,1,float16,fp8,0,0.013969600200653076
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,4,1,128,1,fp8,fp8,0,0.013876800239086152
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,4,2,128,1,float16,float16,0,0.013180799782276154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,4,2,128,1,float16,fp8,0,0.014132800698280334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,4,2,128,1,fp8,fp8,0,0.01404000073671341
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,4,4,128,1,float16,float16,0,0.012854400277137756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,4,4,128,1,float16,fp8,0,0.013681599497795105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,4,4,128,1,fp8,fp8,0,0.013652800023555756
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,4,1,128,1,float16,float16,0,0.01250080019235611
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,4,1,128,1,float16,fp8,0,0.013278399407863618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,4,1,128,1,fp8,fp8,0,0.013300800323486328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,4,2,128,1,float16,float16,0,0.012668800354003907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,4,2,128,1,float16,fp8,0,0.013462400436401368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,4,2,128,1,fp8,fp8,0,0.013225600123405457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,4,4,128,1,float16,float16,0,0.012145599722862244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,4,4,128,1,float16,fp8,0,0.012740799784660339
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,4,4,128,1,fp8,fp8,0,0.012859199941158295
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,4,1,128,1,float16,float16,0,0.011582399904727935
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,4,1,128,1,float16,fp8,0,0.012428800016641617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,4,1,128,1,fp8,fp8,0,0.012347199767827988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,4,2,128,1,float16,float16,0,0.01173119992017746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,4,2,128,1,float16,fp8,0,0.01249919980764389
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,4,2,128,1,fp8,fp8,0,0.012545600533485413
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,4,4,128,1,float16,float16,0,0.011681599915027619
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,4,4,128,1,float16,fp8,0,0.012558400630950928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,4,4,128,1,fp8,fp8,0,0.012680000066757202
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,4,1,128,1,float16,float16,0,0.011673600226640702
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,4,1,128,1,float16,fp8,0,0.012481600046157837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,4,1,128,1,fp8,fp8,0,0.012534399330615998
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,4,2,128,1,float16,float16,0,0.01175839975476265
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,4,2,128,1,float16,fp8,0,0.012558400630950928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,4,2,128,1,fp8,fp8,0,0.012484800070524216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,4,4,128,1,float16,float16,0,0.011526399850845337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,4,4,128,1,float16,fp8,0,0.01234399974346161
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,4,1,128,1,float16,float16,0,0.011393599957227708
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,4,1,128,1,float16,fp8,0,0.012097600102424621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,4,1,128,1,fp8,fp8,0,0.012092799693346024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,4,2,128,1,float16,float16,0,0.011392000317573547
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,4,2,128,1,float16,fp8,0,0.012055999785661697
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,4,2,128,1,fp8,fp8,0,0.012225600332021714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,4,4,128,1,float16,float16,0,0.011505600064992905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,4,4,128,1,float16,fp8,0,0.01239359974861145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,4,4,128,1,fp8,fp8,0,0.012383999675512314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,4,1,128,1,float16,float16,0,0.011606399714946748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,4,1,128,1,float16,fp8,0,0.01226240023970604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,4,1,128,1,fp8,fp8,0,0.012355200201272964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,4,4,128,1,fp8,fp8,0,0.029121598601341246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,4,2,128,1,fp8,fp8,0,0.012353599816560746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,4,4,128,1,float16,float16,0,0.011587200313806533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,4,4,128,1,float16,float16,0,0.011622399836778641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,4,4,128,1,float16,fp8,0,0.012484800070524216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,4,1,128,1,float16,float16,0,0.01144160032272339
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,4,4,128,1,fp8,fp8,0,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,4,1,128,1,float16,fp8,0,0.012049599736928939
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,4,1,128,1,fp8,fp8,0,0.012172800302505494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,4,2,128,1,float16,float16,0,0.011475200206041336
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,4,2,128,1,float16,fp8,0,0.012243200093507767
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,4,2,128,1,fp8,fp8,0,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,4,4,128,1,fp8,fp8,0,0.029769599437713623
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,4,2,128,1,float16,float16,0,0.01154239997267723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,4,2,128,1,float16,fp8,0,0.01242400035262108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,4,4,128,1,float16,fp8,0,0.012222400307655335
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,2,2,128,1,float16,float16,0,0.8880415916442871
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,2,1,128,1,float16,float16,0,1.7072320938110352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,2,2,128,1,float16,fp8,0,0.839140796661377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,2,1,128,1,fp8,fp8,0,1.6222415924072267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,2,2,128,1,fp8,fp8,0,0.8331007957458496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,2,1,128,1,float16,float16,0,0.8576416015625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,2,1,128,1,float16,fp8,0,0.8172991752624512
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,2,2,128,1,float16,fp8,0,0.42667360305786134
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,2,1,128,1,fp8,fp8,0,0.8215840339660645
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,2,2,128,1,fp8,fp8,0,0.436078405380249
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,2,1,128,1,float16,float16,0,0.44654078483581544
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,2,1,128,1,float16,fp8,0,0.4117455959320068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,2,1,128,1,fp8,fp8,0,0.4147679805755615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,2,2,128,1,float16,float16,0,0.24098079204559325
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,2,2,128,1,float16,fp8,0,0.2185375928878784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,2,2,128,1,fp8,fp8,0,0.2182080030441284
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,2,1,128,1,float16,float16,0,0.221230411529541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,2,1,128,1,float16,fp8,0,0.2245743989944458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,2,1,128,1,fp8,fp8,0,0.2161168098449707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,4,4,128,1,fp8,fp8,0,0.012329600006341934
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,2,2,128,1,float16,float16,0,0.5694655895233154
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,2,1,128,1,float16,float16,0,1.0173279762268066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,2,1,128,1,fp8,fp8,0,0.9760512351989746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,2,2,128,1,float16,fp8,0,0.49879040718078616
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,2,2,128,1,fp8,fp8,0,0.4986447811126709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16384,2,1,128,1,float16,fp8,0,1.616147232055664
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,2,1,128,1,float16,float16,0,0.5617728233337402
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,2,1,128,1,float16,fp8,0,0.5047967910766602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,2,2,128,1,float16,float16,0,0.3122704029083252
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,2,2,128,1,float16,fp8,0,0.26708800792694093
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,2,2,128,1,fp8,fp8,0,0.2912512063980103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,2,1,128,1,float16,float16,0,0.2771199941635132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,2,1,128,1,float16,fp8,0,0.26290240287780764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,2,1,128,1,fp8,fp8,0,0.26488480567932127
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,2,2,128,1,float16,float16,0,0.18558239936828613
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,2,2,128,1,float16,fp8,0,0.1678447961807251
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,2,2,128,1,float16,float16,0,0.4468575954437256
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,2,1,128,1,float16,float16,0,0.16923199892044066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,2,1,128,1,fp8,fp8,0,0.16587680578231812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,4,2,128,1,fp8,fp8,0,0.012244799733161926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,2,1,128,1,float16,float16,0,0.7230256080627442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,2,1,128,1,float16,fp8,0,0.6881951808929443
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,10240,2,1,128,1,fp8,fp8,0,0.692193603515625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,2,2,128,1,float16,float16,0,0.409227180480957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,2,2,128,1,fp8,fp8,0,0.3617568016052246
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,2,1,128,1,float16,float16,0,0.3731872081756592
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,2,2,128,1,float16,float16,0,0.243340802192688
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,2,1,128,1,fp8,fp8,0,0.36169281005859377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,12288,2,1,128,1,float16,fp8,0,0.9596320152282715
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,2,2,128,1,fp8,fp8,0,0.19032640457153321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,2,1,128,1,float16,float16,0,0.19726239442825316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,2,1,128,1,float16,fp8,0,0.1894368052482605
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,2,1,128,1,fp8,fp8,0,0.19007999897003175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,2,1,128,1,fp8,fp8,0,0.4966928005218506
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,2,2,128,1,fp8,fp8,0,0.1384992003440857
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,2,1,128,1,float16,float16,0,0.14266560077667237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,2,2,128,1,fp8,fp8,0,0.16550079584121705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,2,1,128,1,float16,fp8,0,0.13882720470428467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,2,1,128,1,fp8,fp8,0,0.13937439918518066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,2,1,128,1,float16,float16,0,0.9166416168212891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,2,1,128,1,float16,fp8,0,0.881100845336914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,2,2,128,1,float16,fp8,0,0.3642672061920166
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,2,1,128,1,float16,fp8,0,0.35852479934692383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,2,2,128,1,float16,fp8,0,0.4491936206817627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,2,2,128,1,float16,fp8,0,0.19283360242843628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,2,2,128,1,fp8,fp8,0,0.44547200202941895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,2,1,128,1,float16,float16,0,0.4649807929992676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,2,1,128,1,float16,fp8,0,0.45093598365783694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,2,2,128,1,float16,float16,0,0.14262399673461915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,2,2,128,1,float16,float16,0,0.2453984022140503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,2,1,128,1,fp8,fp8,0,0.4446767807006836
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,2,2,128,1,float16,fp8,0,0.13819199800491333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,2,2,128,1,float16,fp8,0,0.2313999891281128
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,2,2,128,1,fp8,fp8,0,0.22839839458465577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,2,1,128,1,float16,float16,0,0.23796799182891845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,2,2,128,1,float16,float16,0,0.12610080242156982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,2,2,128,1,float16,fp8,0,0.1196943998336792
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,2,2,128,1,fp8,fp8,0,0.12041599750518799
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,2,1,128,1,float16,float16,0,0.12027839422225953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,2,1,128,1,float16,fp8,0,0.11787840127944946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,2,1,128,1,fp8,fp8,0,0.12003840208053589
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,2,2,128,1,float16,float16,0,0.11424319744110108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,2,2,128,1,float16,fp8,0,0.1130560040473938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,2,2,128,1,fp8,fp8,0,0.1130511999130249
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,2,1,128,1,float16,float16,0,0.11450719833374023
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,2,1,128,1,float16,fp8,0,0.11375839710235595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,2,1,128,1,fp8,fp8,0,0.11283040046691895
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,2,1,128,1,float16,float16,0,0.5837584018707276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,2,1,128,1,float16,fp8,0,0.5611328125
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,6144,2,1,128,1,fp8,fp8,0,0.5660751819610595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,2,2,128,1,float16,float16,0,0.32119359970092776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,2,2,128,1,float16,fp8,0,0.3029200077056885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,2,2,128,1,float16,float16,0,0.484284782409668
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,2,2,128,1,fp8,fp8,0,0.3049488067626953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,2,1,128,1,float16,float16,0,0.3114959955215454
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,8192,2,1,128,1,fp8,fp8,0,0.9019023895263671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,2,1,128,1,float16,fp8,0,0.29991679191589354
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,2,2,128,1,float16,float16,0,0.17615840435028077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,2,2,128,1,float16,fp8,0,0.1670575976371765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,2,1,128,1,fp8,fp8,0,0.3024688005447388
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,2,2,128,1,fp8,fp8,0,0.16535040140151977
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,2,1,128,1,float16,float16,0,0.1715504050254822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,2,1,128,1,float16,fp8,0,0.16655999422073364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,2,1,128,1,fp8,fp8,0,0.16635199785232543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,2,2,128,1,float16,float16,0,0.097980797290802
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,2,2,128,1,float16,fp8,0,0.0934719979763031
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,2,2,128,1,fp8,fp8,0,0.09514560103416443
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,2,1,128,1,float16,float16,0,0.09630240201950073
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,2,1,128,1,fp8,fp8,0,0.09359840154647828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,2,1,128,1,float16,fp8,0,0.2269968032836914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,2,1,128,1,fp8,fp8,0,0.22701120376586914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,2,1,128,1,float16,float16,0,0.06156799793243408
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,2,1,128,1,float16,fp8,0,0.06084160208702087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,2,1,128,1,fp8,fp8,0,0.06175199747085571
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,2,1,128,1,float16,float16,0,0.5603216171264649
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,2,1,128,1,float16,fp8,0,0.5401760101318359
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,2,2,128,1,float16,float16,0,0.30904479026794435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,4096,2,1,128,1,fp8,fp8,0,0.5394256114959717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,2,2,128,1,float16,fp8,0,0.29052960872650146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,2,2,128,1,fp8,fp8,0,0.2902911901473999
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,2,1,128,1,float16,float16,0,0.2974191904067993
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,2,1,128,1,float16,fp8,0,0.28670558929443357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,2,2,128,1,float16,float16,0,0.17381919622421266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,2,1,128,1,fp8,fp8,0,0.28752799034118653
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,2,2,128,1,float16,fp8,0,0.16216640472412108
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,2,2,128,1,fp8,fp8,0,0.161628794670105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,2,1,128,1,float16,float16,0,0.16692960262298584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,2,1,128,1,float16,fp8,0,0.1606927990913391
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,2,2,128,1,float16,float16,0,0.07095040082931518
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,2,2,128,1,float16,float16,0,0.10415840148925781
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,2,2,128,1,float16,fp8,0,0.07000319957733155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,2,2,128,1,fp8,fp8,0,0.09842240214347839
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,2,1,128,1,float16,float16,0,0.100164794921875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,2,1,128,1,float16,fp8,0,0.09730079770088196
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,2,1,128,1,fp8,fp8,0,0.09804959893226624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,2,1,128,1,float16,fp8,0,0.1726207971572876
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,2,2,128,1,float16,fp8,0,0.06671040058135987
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,2,2,128,1,fp8,fp8,0,0.06681600213050842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,2,1,128,1,float16,float16,0,0.06664159893989563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,2,1,128,1,float16,fp8,0,0.06678720116615296
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,2,1,128,1,fp8,fp8,0,0.06704639792442321
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,2,2,128,1,float16,fp8,0,0.04487999975681305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,2,2,128,1,fp8,fp8,0,0.04536640048027039
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,2,1,128,1,float16,float16,0,0.04496960043907165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,2,1,128,1,float16,fp8,0,0.04522559940814972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,2,1,128,1,fp8,fp8,0,0.04475519955158234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,2,1,128,1,float16,float16,0,0.3525520086288452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,2,1,128,1,float16,fp8,0,0.3402767896652222
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,2,2,128,1,float16,float16,0,0.20062720775604248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,3072,2,1,128,1,fp8,fp8,0,0.34342079162597655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,2,2,128,1,float16,fp8,0,0.18863840103149415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,2,2,128,1,fp8,fp8,0,0.18990880250930786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,2,1,128,1,float16,float16,0,0.1918287992477417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,2,1,128,1,float16,fp8,0,0.18572959899902344
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,2,1,128,1,fp8,fp8,0,0.18549920320510865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,2,1,128,1,float16,fp8,0,0.09391199946403503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,2,1,128,1,fp8,fp8,0,0.15975199937820433
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,2,2,128,1,float16,fp8,0,0.09835039973258972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,2,2,128,1,fp8,fp8,0,0.10836000442504883
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,2,1,128,1,float16,float16,0,0.1105728030204773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,2,1,128,1,float16,fp8,0,0.10604640245437622
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,2,1,128,1,fp8,fp8,0,0.1065775990486145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,2,2,128,1,float16,float16,0,0.06669759750366211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,2,2,128,1,float16,float16,0,0.06665279865264892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,2,2,128,1,float16,fp8,0,0.06735039949417114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,2,2,128,1,fp8,fp8,0,0.06701279878616333
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,2,1,128,1,float16,float16,0,0.06498559713363647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,2,1,128,1,float16,fp8,0,0.065583997964859
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,2,1,128,1,fp8,fp8,0,0.06563839912414551
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,2,2,128,1,float16,float16,0,0.05225759744644165
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,2,2,128,1,float16,fp8,0,0.0514735996723175
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,2,2,128,1,fp8,fp8,0,0.05151200294494629
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,2,1,128,1,float16,fp8,0,0.049374398589134214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,2,1,128,1,fp8,fp8,0,0.04899199903011322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,2,2,128,1,float16,fp8,0,0.03212960064411163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,2,2,128,1,fp8,fp8,0,0.031851199269294736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,2,1,128,1,float16,float16,0,0.031411200761795044
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,2,1,128,1,float16,fp8,0,0.031876799464225766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,2,1,128,1,fp8,fp8,0,0.03237279951572418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,2,1,128,1,float16,float16,0,0.3514672040939331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,2,1,128,1,float16,fp8,0,0.34825921058654785
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,2,2,128,1,float16,float16,0,0.20397601127624512
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,2048,2,1,128,1,fp8,fp8,0,0.34465439319610597
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,2,2,128,1,float16,fp8,0,0.18972320556640626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,2,2,128,1,fp8,fp8,0,0.1887231945991516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,2,1,128,1,float16,float16,0,0.192849600315094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,2,1,128,1,float16,fp8,0,0.1845903992652893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,2,2,128,1,float16,float16,0,0.11631360054016113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,2,2,128,1,float16,float16,0,0.11390719413757325
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,2,2,128,1,fp8,fp8,0,0.06909279823303223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,2,2,128,1,fp8,fp8,0,0.10604959726333618
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,2,1,128,1,float16,float16,0,0.10721280574798583
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,2,1,128,1,float16,fp8,0,0.10334559679031372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,2,2,128,1,float16,float16,0,0.04530239999294281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,2,1,128,1,fp8,fp8,0,0.10325759649276733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,2,2,128,1,float16,float16,0,0.03161920011043549
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,2,2,128,1,float16,float16,0,0.07039039731025695
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,2,2,128,1,float16,fp8,0,0.06458079814910889
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,2,2,128,1,fp8,fp8,0,0.06430240273475647
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,2,1,128,1,float16,float16,0,0.06597920060157776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,2,1,128,1,float16,fp8,0,0.06412799954414368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,2,1,128,1,fp8,fp8,0,0.06395040154457092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,2,2,128,1,float16,float16,0,0.041731199622154234
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,2,2,128,1,float16,fp8,0,0.04126560091972351
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,2,2,128,1,fp8,fp8,0,0.04131360054016113
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,2,1,128,1,float16,fp8,0,0.041176000237464906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,2,1,128,1,fp8,fp8,0,0.041300800442695615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,2,2,128,1,float16,fp8,0,0.03311359882354736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,2,2,128,1,fp8,fp8,0,0.033118399977684024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,2,1,128,1,float16,float16,0,0.0324400007724762
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,2,1,128,1,float16,fp8,0,0.03286879956722259
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,2,1,128,1,fp8,fp8,0,0.03287520110607147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,2,2,128,1,float16,float16,0,0.023583999276161192
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,2,2,128,1,float16,fp8,0,0.024401600658893585
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,2,2,128,1,fp8,fp8,0,0.024348799884319306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,2,1,128,1,float16,float16,0,0.023048000037670137
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,2,1,128,1,float16,fp8,0,0.02420639991760254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,2,1,128,1,fp8,fp8,0,0.18663040399551392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,2,1,128,1,float16,float16,0,0.23252320289611816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,2,1,128,1,float16,fp8,0,0.23013598918914796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1536,2,1,128,1,fp8,fp8,0,0.23003358840942384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,2,2,128,1,float16,float16,0,0.13711680173873902
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,2,2,128,1,float16,float16,0,0.03250400125980377
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,2,2,128,1,float16,fp8,0,0.12840160131454467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,2,2,128,1,fp8,fp8,0,0.12871840000152587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,2,1,128,1,float16,float16,0,0.12888480424880983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,2,1,128,1,float16,fp8,0,0.12497919797897339
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,2,1,128,1,fp8,fp8,0,0.12521920204162598
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,2,2,128,1,float16,float16,0,0.07896639704704285
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,2,2,128,1,float16,fp8,0,0.07460160255432129
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,2,2,128,1,fp8,fp8,0,0.07495679855346679
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,2,1,128,1,float16,float16,0,0.07396320104599
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,2,1,128,1,float16,fp8,0,0.07206879854202271
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,2,1,128,1,fp8,fp8,0,0.0722432017326355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,2,2,128,1,float16,fp8,0,0.04809759855270386
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,2,2,128,1,fp8,fp8,0,0.047881600260734555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,2,1,128,1,fp8,fp8,0,0.02417760044336319
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,2,1,128,1,float16,fp8,0,0.04748319983482361
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,2,1,128,1,fp8,fp8,0,0.04753279983997345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,2,2,128,1,float16,fp8,0,0.040531200170516965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,2,2,128,1,fp8,fp8,0,0.04054720103740692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,2,1,128,1,float16,float16,0,0.03931199908256531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,2,1,128,1,float16,float16,0,0.04877119958400726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,2,1,128,1,fp8,fp8,0,0.040479999780654904
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,2,2,128,1,float16,float16,0,0.02608320116996765
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,2,2,128,1,float16,fp8,0,0.026438400149345398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,2,2,128,1,fp8,fp8,0,0.026470398902893065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,2,1,128,1,float16,float16,0,0.04139359891414642
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,2,1,128,1,float16,float16,0,0.025124800205230714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,2,1,128,1,float16,fp8,0,0.026464000344276428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,2,1,128,1,fp8,fp8,0,0.02655999958515167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,2,2,128,1,float16,float16,0,0.02072319984436035
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,2,2,128,1,float16,fp8,0,0.02040639966726303
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,2,2,128,1,fp8,fp8,0,0.02028159946203232
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,2,1,128,1,float16,fp8,0,0.020308800041675568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,2,1,128,1,fp8,fp8,0,0.02043039947748184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,2,1,128,1,float16,float16,0,0.24847519397735596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,2,2,128,1,float16,float16,0,0.047572800517082216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,2,1,128,1,float16,fp8,0,0.25124800205230713
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,2,2,128,1,float16,fp8,0,0.10808320045471191
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1024,2,1,128,1,fp8,fp8,0,0.2519295930862427
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,2,2,128,1,float16,float16,0,0.04006560146808624
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,2,2,128,1,float16,float16,0,0.14608800411224365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,2,2,128,1,float16,fp8,0,0.140228796005249
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,2,2,128,1,fp8,fp8,0,0.14097440242767334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,2,1,128,1,float16,fp8,0,0.13582719564437867
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,2,2,128,1,float16,float16,0,0.0826640009880066
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,2,1,128,1,float16,fp8,0,0.04047360122203827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,2,2,128,1,float16,fp8,0,0.07971040010452271
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,2,2,128,1,fp8,fp8,0,0.07952319979667663
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,2,1,128,1,float16,float16,0,0.07593920230865478
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,2,1,128,1,float16,fp8,0,0.07627360224723816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,2,1,128,1,fp8,fp8,0,0.07641599774360656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,2,2,128,1,float16,float16,0,0.05024319887161255
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,2,2,128,1,float16,fp8,0,0.04848000109195709
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,2,2,128,1,fp8,fp8,0,0.04822719991207123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,2,1,128,1,float16,float16,0,0.046209600567817685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,2,1,128,1,float16,fp8,0,0.046983999013900754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,2,1,128,1,fp8,fp8,0,0.04705440104007721
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,2,2,128,1,float16,float16,0,0.028935998678207397
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,2,2,128,1,fp8,fp8,0,0.02991040050983429
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,2,1,128,1,float16,float16,0,0.02877120077610016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,2,1,128,1,float16,fp8,0,0.02964639961719513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,2,1,128,1,fp8,fp8,0,0.029500800371170043
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,2,2,128,1,float16,float16,0,0.026332798600196838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,2,1,128,1,float16,float16,0,0.020611199736595153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,2,2,128,1,float16,fp8,0,0.02741119861602783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,2,2,128,1,fp8,fp8,0,0.02759360074996948
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,2,1,128,1,float16,float16,0,0.026105600595474242
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,2,1,128,1,float16,fp8,0,0.02751680016517639
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,2,1,128,1,fp8,fp8,0,0.027489599585533143
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,2,2,128,1,float16,float16,0,0.021529600024223328
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,2,2,128,1,float16,fp8,0,0.02099359929561615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,2,2,128,1,fp8,fp8,0,0.021190400421619415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,2,1,128,1,float16,float16,0,0.021116800606250763
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,2,1,128,1,float16,fp8,0,0.021059200167655945
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,2,1,128,1,fp8,fp8,0,0.021132799983024596
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,2,2,128,1,float16,float16,0,0.018403199315071107
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,2,2,128,1,float16,fp8,0,0.01955839991569519
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,2,2,128,1,fp8,fp8,0,0.01945279985666275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,2,1,128,1,float16,float16,0,0.01828960031270981
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,2,1,128,1,fp8,fp8,0,0.019871999323368073
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,2,1,128,1,float16,float16,0,0.04636319875717163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,2,1,128,1,float16,float16,0,0.17009600400924682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,2,1,128,1,float16,fp8,0,0.1780192017555237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,512,2,1,128,1,fp8,fp8,0,0.17842559814453124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,2,1,128,1,float16,float16,0,0.13461120128631593
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,2,2,128,1,float16,fp8,0,0.10230400562286376
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,2,1,128,1,fp8,fp8,0,0.1355504035949707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,2,1,128,1,float16,float16,0,0.09243680238723755
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,2,1,128,1,float16,fp8,0,0.09700639843940735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,2,1,128,1,fp8,fp8,0,0.09646720290184022
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,2,2,128,1,float16,float16,0,0.06054400205612183
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,2,2,128,1,float16,fp8,0,0.058220797777175905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,2,2,128,1,fp8,fp8,0,0.05855839848518372
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,2,1,128,1,float16,float16,0,0.05393760204315186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,2,1,128,1,float16,fp8,0,0.0546720027923584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,2,1,128,1,fp8,fp8,0,0.054897600412368776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,2,2,128,1,float16,float16,0,0.03628639876842499
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,2,2,128,1,float16,fp8,0,0.034771201014518735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,2,2,128,1,fp8,fp8,0,0.03482080101966858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,2,1,128,1,float16,float16,0,0.031436800956726074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,2,1,128,1,float16,fp8,0,0.0334991991519928
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,2,1,128,1,fp8,fp8,0,0.03329440057277679
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,2,2,128,1,float16,float16,0,0.02008640021085739
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,2,2,128,1,float16,fp8,0,0.021515199542045595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,2,2,128,1,fp8,fp8,0,0.02155359983444214
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,2,1,128,1,float16,float16,0,0.02005600035190582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,2,1,128,1,float16,fp8,0,0.02146400064229965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,2,1,128,1,fp8,fp8,0,0.021478399634361267
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,2,2,128,1,float16,float16,0,0.018483200669288637
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,2,2,128,1,float16,fp8,0,0.0200655996799469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,2,1,128,1,float16,float16,0,0.0181551992893219
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,2,1,128,1,float16,fp8,0,0.019785599410533906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,2,1,128,1,fp8,fp8,0,0.019651199877262115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,2,2,128,1,float16,float16,0,0.0177279993891716
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,2,2,128,1,float16,fp8,0,0.019123199582099914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,2,1,128,1,float16,fp8,0,0.019529600441455842
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,2,2,128,1,fp8,fp8,0,0.019068799912929535
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,2,1,128,1,float16,float16,0,0.017371200025081635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,2,1,128,1,float16,fp8,0,0.018972800672054292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,2,1,128,1,fp8,fp8,0,0.019273599982261656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,2,2,128,1,float16,float16,0,0.017110399901866913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,2,2,128,1,float16,fp8,0,0.019047999382019044
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,2,1,128,1,float16,float16,0,0.01698399931192398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,2,1,128,1,float16,fp8,0,0.019011199474334717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,2,1,128,1,fp8,fp8,0,0.018982400000095368
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,2,2,128,1,float16,float16,0,0.10444320440292358
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,2,2,128,1,float16,fp8,0,0.018796800076961516
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,2,2,128,1,fp8,fp8,0,0.10207040309906006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,2,2,128,1,fp8,fp8,0,0.018827199935913086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,2,1,128,1,float16,fp8,0,0.018592000007629395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,2,1,128,1,fp8,fp8,0,0.01870400011539459
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,2,1,128,1,float16,float16,0,0.08208640217781067
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,2,1,128,1,float16,fp8,0,0.08590559959411621
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,2,2,128,1,float16,fp8,0,0.02983039915561676
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,2,2,128,1,float16,float16,0,0.05466560125350952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,2,2,128,1,float16,fp8,0,0.04925439953804016
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,2,1,128,1,float16,float16,0,0.0468207985162735
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,2,2,128,1,fp8,fp8,0,0.04922879934310913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,2,2,128,1,fp8,fp8,0,0.020216000080108643
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,2,1,128,1,fp8,fp8,0,0.047884801030159
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,2,2,128,1,float16,fp8,0,0.028934401273727418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,2,2,128,1,fp8,fp8,0,0.02903040051460266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,2,1,128,1,float16,float16,0,0.02656959891319275
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,2,1,128,1,float16,fp8,0,0.028436800837516783
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,2,1,128,1,fp8,fp8,0,0.02836799919605255
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,2,2,128,1,float16,float16,0,0.017265599966049195
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,2,2,128,1,float16,fp8,0,0.10682560205459594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,2,2,128,1,float16,fp8,0,0.018192000687122345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,2,2,128,1,fp8,fp8,0,0.018940800428390504
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,2,2,128,1,fp8,fp8,0,0.018438400328159334
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,2,1,128,1,float16,float16,0,0.017118400335311888
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,2,1,128,1,float16,fp8,0,0.018433600664138794
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,2,1,128,1,fp8,fp8,0,0.018356800079345703
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,2,2,128,1,float16,float16,0,0.015958400070667268
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,2,2,128,1,float16,fp8,0,0.016420799493789672
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,2,2,128,1,fp8,fp8,0,0.016300800442695617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,2,1,128,1,float16,float16,0,0.015587200224399567
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,2,1,128,1,float16,fp8,0,0.016422399878501893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,2,1,128,1,fp8,fp8,0,0.01632480025291443
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,2,2,128,1,float16,float16,0,0.01671839952468872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,2,2,128,1,float16,float16,0,0.01496800035238266
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,2,2,128,1,fp8,fp8,0,0.01557759940624237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,2,1,128,1,float16,float16,0,0.014644800126552582
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,2,1,128,1,fp8,fp8,0,0.015483200550079346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,2,2,128,1,float16,float16,0,0.01404000073671341
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,2,2,128,1,float16,fp8,0,0.015065599977970124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,2,2,128,1,fp8,fp8,0,0.015080000460147857
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,2,1,128,1,float16,float16,0,0.013996799290180207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,2,1,128,1,float16,fp8,0,0.014988799393177033
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,2,1,128,1,fp8,fp8,0,0.015136000514030457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,2,2,128,1,float16,float16,0,0.013851200044155122
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,2,2,128,1,float16,fp8,0,0.014956800639629364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,2,2,128,1,fp8,fp8,0,0.014864000678062438
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,2,1,128,1,float16,float16,0,0.013716800510883332
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,256,2,1,128,1,fp8,fp8,0,0.08602560162544251
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,2,1,128,1,fp8,fp8,0,0.014936000108718872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,2,2,128,1,float16,float16,0,0.0135343998670578
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,2,2,128,1,float16,fp8,0,0.014579200744628906
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,2,2,128,1,fp8,fp8,0,0.01459839940071106
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,2,1,128,1,float16,float16,0,0.013420799374580383
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,2,1,128,1,float16,fp8,0,0.014604799449443817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,2,1,128,1,fp8,fp8,0,0.014580799639225006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,2,2,128,1,float16,float16,0,0.03194240033626557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,2,1,128,1,float16,float16,0,0.04527679979801178
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,2,1,128,1,float16,fp8,0,0.044721600413322446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,128,2,1,128,1,fp8,fp8,0,0.04438399970531463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,2,2,128,1,float16,float16,0,0.030041599273681642
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,2,2,128,1,float16,fp8,0,0.02577440142631531
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,2,2,128,1,fp8,fp8,0,0.02561599910259247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,2,1,128,1,float16,float16,0,0.02462559938430786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,2,1,128,1,float16,fp8,0,0.02545759975910187
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,2,1,128,1,fp8,fp8,0,0.025518399477005006
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,2,2,128,1,float16,float16,0,0.017057600617408752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,2,2,128,1,float16,fp8,0,0.01732800006866455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,2,2,128,1,fp8,fp8,0,0.017211200296878816
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,2,1,128,1,float16,float16,0,0.016891199350357055
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,2,1,128,1,float16,fp8,0,0.017364799976348877
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,2,1,128,1,fp8,fp8,0,0.01722240000963211
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,2,2,128,1,float16,float16,0,0.01438080072402954
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,2,2,128,1,float16,fp8,0,0.014815999567508698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,2,2,128,1,fp8,fp8,0,0.01464959979057312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,2,1,128,1,float16,float16,0,0.014326399564743042
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,2,1,128,1,float16,fp8,0,0.014684799313545226
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,2,2,128,1,float16,fp8,0,0.015617600083351136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,2,1,128,1,float16,fp8,0,0.01563519984483719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,2,2,128,1,float16,fp8,0,0.013788799941539764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,2,2,128,1,fp8,fp8,0,0.01356479972600937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,2,1,128,1,float16,float16,0,0.013185599446296692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,2,1,128,1,float16,fp8,0,0.01340160071849823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,2,1,128,1,float16,float16,0,0.016606399416923524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,2,2,128,1,float16,float16,0,0.012665599584579468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,2,2,128,1,float16,fp8,0,0.013124799728393555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,2,2,128,1,fp8,fp8,0,0.012798400223255157
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,2,1,128,1,float16,float16,0,0.012332800030708312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,2,1,128,1,fp8,fp8,0,0.01284160017967224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,2,2,128,1,float16,float16,0,0.012164799869060517
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,2,2,128,1,float16,fp8,0,0.012748800218105316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,2,2,128,1,fp8,fp8,0,0.012814399600028992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,2,1,128,1,float16,float16,0,0.01223360002040863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,2,1,128,1,float16,fp8,0,0.01284320056438446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,2,1,128,1,fp8,fp8,0,0.01281760036945343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,2,2,128,1,float16,float16,0,0.011998400092124939
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,2,2,128,1,float16,fp8,0,0.012505599856376648
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,2,2,128,1,fp8,fp8,0,0.012460800260305405
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,2,1,128,1,float16,float16,0,0.011913599818944931
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,2,1,128,1,float16,fp8,0,0.01250080019235611
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,2,1,128,1,fp8,fp8,0,0.012508800625801087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,2,2,128,1,float16,float16,0,0.01194079965353012
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,2,2,128,1,float16,fp8,0,0.012595200538635254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,2,2,128,1,fp8,fp8,0,0.012564800679683685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,2,1,128,1,float16,float16,0,0.011868800222873687
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,2,1,128,1,float16,fp8,0,0.012518399953842163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,2,1,128,1,fp8,fp8,0,0.012414400279521943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,2,1,128,1,float16,fp8,0,0.028249600529670717
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,2,1,128,1,fp8,fp8,0,0.02823520004749298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,2,1,128,1,fp8,fp8,0,0.01480959951877594
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,2,2,128,1,float16,float16,0,0.02065120041370392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,2,2,128,1,float16,fp8,0,0.02276480048894882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,2,2,128,1,fp8,fp8,0,0.022468799352645875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,2,1,128,1,float16,float16,0,0.017907199263572694
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,2,1,128,1,fp8,fp8,0,0.014028799533843995
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,2,1,128,1,fp8,fp8,0,0.018294399976730345
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,2,2,128,1,float16,float16,0,0.014796799421310425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,2,2,128,1,float16,fp8,0,0.015292799472808838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,2,2,128,1,fp8,fp8,0,0.015315200388431548
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,2,1,128,1,float16,float16,0,0.01478240042924881
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,2,1,128,1,float16,fp8,0,0.015379199385643005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,2,2,128,1,float16,float16,0,0.013236799836158752
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,2,2,128,1,float16,fp8,0,0.014023999869823455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,2,2,128,1,fp8,fp8,0,0.014001600444316864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,2,1,128,1,float16,float16,0,0.013348799943923951
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,2,1,128,1,float16,fp8,0,0.014254400134086609
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,2,1,128,1,fp8,fp8,0,0.01424960047006607
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,2,2,128,1,float16,float16,0,0.012755200266838074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,2,2,128,1,float16,fp8,0,0.013099199533462525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,2,1,128,1,float16,fp8,0,0.04778720140457153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,2,2,128,1,fp8,fp8,0,0.013247999548912048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,2,1,128,1,float16,float16,0,0.012484800070524216
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,2,1,128,1,float16,fp8,0,0.01327199935913086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,2,1,128,1,fp8,fp8,0,0.013220800459384919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,2,2,128,1,float16,fp8,0,0.013099199533462525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,2,2,128,1,fp8,fp8,0,0.013046400249004364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,2,1,128,1,float16,float16,0,0.012363199889659882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,2,1,128,1,float16,fp8,0,0.013035200536251068
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,2,1,128,1,fp8,fp8,0,0.012992000579833985
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,2,2,128,1,float16,float16,0,0.01183359995484352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,2,2,128,1,float16,fp8,0,0.012601600587368011
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,2,2,128,1,fp8,fp8,0,0.012647999823093415
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,64,2,1,128,1,float16,float16,0,0.026638400554656983
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,2,1,128,1,float16,float16,0,0.012116800248622894
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,2,2,128,1,float16,float16,0,0.013494400680065155
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,2,1,128,1,float16,fp8,0,0.012779200077056884
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,2,1,128,1,fp8,fp8,0,0.012516799569129943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,2,2,128,1,float16,float16,0,0.011617600172758102
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,2,2,128,1,float16,fp8,0,0.01250240057706833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,2,2,128,1,fp8,fp8,0,0.01223199963569641
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,2,1,128,1,float16,fp8,0,0.018111999332904815
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,2,1,128,1,float16,float16,0,0.01199679970741272
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,2,1,128,1,float16,fp8,0,0.012408000230789185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,2,1,128,1,fp8,fp8,0,0.012520000338554382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,2,2,128,1,float16,fp8,0,0.012219200283288956
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,2,2,128,1,fp8,fp8,0,0.012368000298738479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,2,1,128,1,float16,float16,0,0.011748799681663513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,2,1,128,1,float16,fp8,0,0.01250080019235611
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,2,1,128,1,fp8,fp8,0,0.01541599929332733
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,2,1,128,1,float16,float16,0,0.023127999901771546
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,2,1,128,1,float16,fp8,0,0.026046401262283324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,32,2,1,128,1,fp8,fp8,0,0.024910399317741395
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,2,1,128,1,float16,fp8,0,0.014851200580596923
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,2,2,128,1,float16,fp8,0,0.02141599953174591
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,2,2,128,1,fp8,fp8,0,0.02120320051908493
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,2,1,128,1,float16,float16,0,0.016275200247764587
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,2,1,128,1,float16,fp8,0,0.0168272003531456
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,2,1,128,1,fp8,fp8,0,0.017136000096797943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,2,2,128,1,float16,float16,0,0.014260800182819366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,2,2,128,1,float16,fp8,0,0.014703999459743499
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,2,2,128,1,fp8,fp8,0,0.014867199957370758
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,2,1,128,1,float16,float16,0,0.013753600418567657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,2,1,128,1,float16,fp8,0,0.014763200283050537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,2,1,128,1,fp8,fp8,0,0.014552000164985656
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,2,2,128,1,float16,float16,0,0.013128000497817992
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,2,2,128,1,float16,fp8,0,0.013551999628543854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,2,2,128,1,fp8,fp8,0,0.013764800131320953
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,2,1,128,1,float16,float16,0,0.012638400495052337
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,2,2,128,1,float16,float16,0,0.011988800019025803
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,2,1,128,1,fp8,fp8,0,0.013385599851608277
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,2,2,128,1,float16,float16,0,0.012059199810028075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,2,2,128,1,float16,fp8,0,0.013046400249004364
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,2,2,128,1,fp8,fp8,0,0.012863999605178833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,2,1,128,1,float16,float16,0,0.012187200039625168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,2,1,128,1,float16,fp8,0,0.01273919939994812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,2,2,128,1,float16,float16,0,0.011884800344705581
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,2,2,128,1,float16,fp8,0,0.012724800407886505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,2,2,128,1,fp8,fp8,0,0.012742400169372559
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,2,1,128,1,float16,float16,0,0.011697600036859513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,2,1,128,1,float16,fp8,0,0.012470400333404541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,2,1,128,1,fp8,fp8,0,0.01247519999742508
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,2,2,128,1,float16,float16,0,0.011723200231790543
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,2,2,128,1,float16,fp8,0,0.012478400021791458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,2,2,128,1,fp8,fp8,0,0.01247360035777092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,2,1,128,1,float16,float16,0,0.01173280030488968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,2,1,128,1,float16,fp8,0,0.012564800679683685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,2,1,128,1,fp8,fp8,0,0.012414400279521943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,2,2,128,1,float16,float16,0,0.01148959994316101
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,2,2,128,1,float16,float16,0,0.011686400324106217
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,2,2,128,1,float16,fp8,0,0.01225920021533966
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,2,1,128,1,float16,fp8,0,0.013265599310398103
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,2,1,128,1,fp8,fp8,0,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,2,1,128,1,float16,fp8,0,0.012531200051307678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,2,1,128,1,fp8,fp8,0,0.012455999851226807
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,2,2,128,1,float16,float16,0,0.011519999802112579
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,2,2,128,1,float16,fp8,0,0.012427199631929398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,2,2,128,1,fp8,fp8,0,0.01223519966006279
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,2,1,128,1,float16,float16,0,0.011641599982976914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,2,1,128,1,float16,fp8,0,0.012238399684429168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,2,1,128,1,fp8,fp8,0,0.012404800206422806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,2,1,128,1,float16,float16,0,0.022177599370479584
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,2,1,128,1,float16,fp8,0,0.024166400730609893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,16,2,1,128,1,fp8,fp8,0,0.02337760031223297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,2,2,128,1,float16,fp8,0,0.02021760046482086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,2,2,128,1,fp8,fp8,0,0.02046079933643341
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,2,1,128,1,float16,float16,0,0.015134400129318238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,2,1,128,1,float16,fp8,0,0.015928000211715698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,2,1,128,1,float16,fp8,0,0.013551999628543854
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,2,2,128,1,float16,float16,0,0.013976000249385834
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,2,2,128,1,float16,fp8,0,0.014601600170135499
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,2,2,128,1,fp8,fp8,0,0.014472000300884247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,2,1,128,1,float16,float16,0,0.013916799426078796
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,2,1,128,1,float16,fp8,0,0.014310400187969207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,2,1,128,1,fp8,fp8,0,0.01451680064201355
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,2,2,128,1,float16,float16,0,0.012972800433635712
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,2,2,128,1,float16,fp8,0,0.01361439973115921
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,2,2,128,1,fp8,fp8,0,0.013665600121021271
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,2,1,128,1,float16,fp8,0,0.013414399325847625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,2,1,128,1,fp8,fp8,0,0.013415999710559845
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,2,2,128,1,float16,float16,0,0.01194560006260872
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,2,2,128,1,float16,fp8,0,0.012758399546146392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,2,2,128,1,fp8,fp8,0,0.012750400602817536
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,2,1,128,1,float16,float16,0,0.011990399658679962
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,2,1,128,1,float16,fp8,0,0.012796799838542938
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,2,1,128,1,fp8,fp8,0,0.012774400413036346
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,2,2,128,1,float16,float16,0,0.011897599697113037
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,2,2,128,1,float16,fp8,0,0.01265919953584671
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,2,2,128,1,fp8,fp8,0,0.012588800489902496
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,2,1,128,1,float16,float16,0,0.011700800061225891
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,2,1,128,1,float16,fp8,0,0.01239520013332367
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,2,2,128,1,float16,float16,0,0.019622400403022766
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,2,1,128,1,fp8,fp8,0,0.01241919994354248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,2,2,128,1,float16,float16,0,0.01173119992017746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,2,2,128,1,float16,fp8,0,0.012371200323104858
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,2,2,128,1,float16,float16,0,0.018780800700187682
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,2,1,128,1,float16,float16,0,0.011528000235557556
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,2,1,128,1,float16,fp8,0,0.012247999757528305
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,2,1,128,1,fp8,fp8,0,0.01226079985499382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,2,2,128,1,float16,float16,0,0.011523199826478958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,2,2,128,1,float16,fp8,0,0.012366399914026261
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,2,1,128,1,fp8,fp8,0,0.016011199355125426
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,2,2,128,1,fp8,fp8,0,0.012441600114107132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,2,1,128,1,float16,float16,0,0.01157120019197464
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,2,1,128,1,fp8,fp8,0,0.01223360002040863
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,2,1,128,1,fp8,fp8,0,0.013011200726032257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,2,2,128,1,float16,float16,0,0.011289600282907486
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,2,2,128,1,fp8,fp8,0,0.012110400199890136
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,2,1,128,1,float16,float16,0,0.011380799859762192
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,2,1,128,1,float16,fp8,0,0.012134400010108948
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,2,1,128,1,fp8,fp8,0,0.01226079985499382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,2,1,128,1,float16,float16,0,0.02112320065498352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,2,1,128,1,float16,fp8,0,0.022785599529743194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,256,1,2,1,128,1,fp8,fp8,0,0.02280319929122925
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,2,2,128,1,float16,float16,0,0.018409599363803864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,2,2,128,1,float16,fp8,0,0.020001600682735442
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,2,2,128,1,fp8,fp8,0,0.019841599464416503
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,2,1,128,1,float16,float16,0,0.01451359987258911
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,2,1,128,1,float16,fp8,0,0.01557600051164627
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,2,1,128,1,fp8,fp8,0,0.015591999888420105
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,2,2,128,1,fp8,fp8,0,0.012319999933242797
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,2,2,128,1,float16,float16,0,0.013574400544166565
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,2,2,128,1,float16,fp8,0,0.014135999977588654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,2,1,128,1,float16,float16,0,0.01313440054655075
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,2,1,128,1,float16,fp8,0,0.013891200721263885
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,2,1,128,1,fp8,fp8,0,0.013924799859523773
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,2,2,128,1,float16,float16,0,0.012622399628162384
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,2,2,128,1,float16,fp8,0,0.01345600038766861
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,2,2,128,1,fp8,fp8,0,0.013553600013256072
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,2,2,128,1,fp8,fp8,0,0.012348800152540206
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,2,1,128,1,float16,float16,0,0.012771199643611907
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,2,1,128,1,float16,fp8,0,0.01345600038766861
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,2,2,128,1,float16,float16,0,0.01191679984331131
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,2,2,128,1,float16,fp8,0,0.012913599610328674
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,2,2,128,1,fp8,fp8,0,0.012931199371814727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,2,1,128,1,float16,float16,0,0.011896000057458878
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,2,1,128,1,float16,fp8,0,0.012564800679683685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,2,1,128,1,float16,fp8,0,0.01236959993839264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,2,1,128,1,fp8,fp8,0,0.012462399899959564
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,2,2,128,1,float16,fp8,0,0.012518399953842163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,2,2,128,1,fp8,fp8,0,0.012361600250005721
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,2,1,128,1,float16,float16,0,0.011558400094509124
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,2,1,128,1,float16,fp8,0,0.01255040019750595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,2,2,128,1,float16,fp8,0,0.012263999879360199
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,2,2,128,1,float16,float16,0,0.011630400270223617
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,2,2,128,1,float16,fp8,0,0.012275200337171555
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,2,1,128,1,float16,float16,0,0.012707200646400452
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,2,1,128,1,float16,float16,0,0.011553599685430526
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,2,1,128,1,float16,fp8,0,0.012363199889659882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,2,1,128,1,fp8,fp8,0,0.012302400171756744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,2,2,128,1,float16,float16,0,0.011499200016260147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,2,2,128,1,float16,fp8,0,0.012036799639463424
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,2,2,128,1,fp8,fp8,0,0.01215839982032776
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,2,1,128,1,float16,fp8,0,0.012217599898576736
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,2,1,128,1,fp8,fp8,0,0.01231200024485588
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,2,2,128,1,float16,float16,0,0.011537600308656693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,2,2,128,1,float16,fp8,0,0.012296000123023986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,2,2,128,1,fp8,fp8,0,0.012160000205039979
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,2,1,128,1,float16,float16,0,0.011433599889278412
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,2,1,128,1,float16,fp8,0,0.012201599776744843
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,2,1,128,1,float16,float16,0,0.011348800361156463
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,2,1,128,1,fp8,fp8,0,0.012169600278139115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,2,2,128,1,fp8,fp8,0,0.014519999921321868
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,1,1,128,1,float16,float16,0,0.4580751895904541
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,1,1,128,1,fp8,fp8,0,0.4380368232727051
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,1,1,128,1,float16,float16,0,0.23360960483551024
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,2,1,128,1,fp8,fp8,0,0.013516800105571746
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,1,1,128,1,fp8,fp8,0,0.22497599124908446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,1,1,128,1,float16,float16,0,0.21698238849639892
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,1,1,128,1,float16,fp8,0,0.21611039638519286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16384,1,1,128,1,fp8,fp8,0,0.2133375883102417
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,2,2,128,1,float16,float16,0,0.011588799953460693
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,1,1,128,1,float16,float16,0,0.3252351999282837
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,1,1,128,1,float16,fp8,0,0.31000800132751466
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,12288,1,1,128,1,fp8,fp8,0,0.3101632118225098
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,2,2,128,1,fp8,fp8,0,0.01231039986014366
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,1,1,128,1,float16,float16,0,0.18142240047454833
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,1,1,128,1,float16,fp8,0,0.1735311985015869
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,12288,1,1,128,1,fp8,fp8,0,0.17346400022506714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,1,1,128,1,float16,fp8,0,0.10050239562988281
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,2,1,128,1,float16,float16,0,0.011296000331640244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,1,1,128,1,float16,float16,0,0.2535264015197754
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,1,1,128,1,float16,fp8,0,0.2435215950012207
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,10240,1,1,128,1,fp8,fp8,0,0.24325599670410156
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,1,1,128,1,float16,float16,0,0.14988640546798707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,1,1,128,1,float16,fp8,0,0.1457327961921692
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,10240,1,1,128,1,fp8,fp8,0,0.14591840505599976
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,1,1,128,1,float16,float16,0,0.08678560256958008
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,1,1,128,1,float16,fp8,0,0.08527839779853821
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,10240,1,1,128,1,fp8,fp8,0,0.08520640134811401
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,1,1,128,1,float16,float16,0,0.3039344072341919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,1,1,128,1,float16,fp8,0,0.2922384023666382
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,8192,1,1,128,1,fp8,fp8,0,0.2928112030029297
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16384,1,1,128,1,float16,fp8,0,0.43965921401977537
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,1,1,128,1,float16,float16,0,0.18479199409484864
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,1,1,128,1,float16,fp8,0,0.17642560005187988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,1,1,128,1,float16,float16,0,0.11988320350646972
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,8192,1,1,128,1,fp8,fp8,0,0.17787200212478638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,1,1,128,1,float16,fp8,0,0.11987520456314087
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16384,1,1,128,1,float16,fp8,0,0.22398879528045654
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,8192,1,1,128,1,fp8,fp8,0,0.11964960098266601
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,1,1,128,1,fp8,fp8,0,0.07162879705429077
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,1,1,128,1,float16,float16,0,0.19565279483795167
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,1,1,128,1,float16,fp8,0,0.18788959980010986
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,6144,1,1,128,1,fp8,fp8,0,0.1885632038116455
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,1,1,128,1,float16,float16,0,0.11376800537109374
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,1,1,128,1,float16,fp8,0,0.11158560514450074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,2,1,128,1,fp8,fp8,0,0.012478400021791458
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,1,1,128,1,float16,float16,0,0.07195519804954528
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,1,1,128,1,float16,fp8,0,0.07118080258369446
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,6144,1,1,128,1,fp8,fp8,0,0.07131680250167846
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,1,1,128,1,float16,float16,0,0.045044800639152525
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,1,1,128,1,float16,fp8,0,0.044628798961639404
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,6144,1,1,128,1,fp8,fp8,0,0.04477919936180115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,1,1,128,1,float16,float16,0,0.18535840511322021
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,1,1,128,1,float16,fp8,0,0.1785423994064331
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,4096,1,1,128,1,fp8,fp8,0,0.17883199453353882
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,1,1,128,1,float16,float16,0,0.11274720430374145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,1,1,128,1,float16,fp8,0,0.10847840309143067
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,4096,1,1,128,1,fp8,fp8,0,0.10841920375823974
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,1,1,128,1,float16,float16,0,0.06912000179290771
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,1,1,128,1,fp8,fp8,0,0.06896479725837708
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,1,1,128,1,float16,float16,0,0.046209600567817685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,1,1,128,1,float16,fp8,0,0.046079999208450316
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,4096,1,1,128,1,fp8,fp8,0,0.04604960083961487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,1,1,128,1,float16,float16,0,0.029854398965835572
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,1,1,128,1,float16,float16,0,0.07184960246086121
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,1,1,128,1,float16,fp8,0,0.030700799822807313
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,4096,1,1,128,1,fp8,fp8,0,0.030737599730491637
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,1,1,128,1,float16,float16,0,0.1232975959777832
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,1,1,128,1,float16,fp8,0,0.11931999921798705
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,1,1,128,1,float16,float16,0,0.07422559857368469
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,1,1,128,1,float16,fp8,0,0.07646399736404419
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,6144,1,1,128,1,fp8,fp8,0,0.11172959804534913
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,3072,1,1,128,1,fp8,fp8,0,0.0759872019290924
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,1,1,128,1,float16,float16,0,0.05350720286369324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,1,1,128,1,float16,float16,0,0.10400639772415161
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,12288,1,1,128,1,fp8,fp8,0,0.10032000541687011
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,1,1,128,1,float16,fp8,0,0.03310079872608185
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,1,1,128,1,fp8,fp8,0,0.03296479880809784
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,1,1,128,1,float16,float16,0,0.024259200692176817
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,1,1,128,1,float16,fp8,0,0.024403199553489685
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,3072,1,1,128,1,fp8,fp8,0,0.02452639937400818
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,4096,1,1,128,1,float16,fp8,0,0.06882399916648865
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,1,1,128,1,float16,float16,0,0.12093440294265748
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,1,1,128,1,float16,fp8,0,0.11732159852981568
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,2048,1,1,128,1,fp8,fp8,0,0.11913919448852539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,1,1,128,1,float16,fp8,0,0.07064639925956726
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,1,1,128,1,fp8,fp8,0,0.07156800031661988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,1,1,128,1,float16,float16,0,0.04412319958209991
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,1,1,128,1,float16,fp8,0,0.0440447986125946
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,2048,1,1,128,1,fp8,fp8,0,0.04393920004367828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,1,1,128,1,float16,float16,0,0.03385440111160278
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,1,1,128,1,float16,fp8,0,0.03421440124511719
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,2048,1,1,128,1,fp8,fp8,0,0.0343423992395401
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,1,1,128,1,float16,float16,0,0.023548799753189086
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,1,1,128,1,float16,fp8,0,0.0250463992357254
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,2048,1,1,128,1,fp8,fp8,0,0.02497120052576065
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,1,1,128,1,float16,fp8,0,0.020046399533748628
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,3072,1,1,128,1,fp8,fp8,0,0.12004799842834472
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,1,1,128,1,float16,float16,0,0.08394240140914917
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,1,1,128,1,float16,fp8,0,0.08317279815673828
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,1,1,128,1,fp8,fp8,0,0.05346879959106445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1536,1,1,128,1,fp8,fp8,0,0.08304319977760315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,3072,1,1,128,1,float16,float16,0,0.03240480124950409
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,1,1,128,1,float16,float16,0,0.051937597990036014
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,1,1,128,1,float16,fp8,0,0.05204319953918457
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1536,1,1,128,1,fp8,fp8,0,0.05207359790802002
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,1,1,128,1,float16,float16,0,0.04140959978103638
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,1,1,128,1,float16,fp8,0,0.04240480065345764
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1536,1,1,128,1,fp8,fp8,0,0.04253920018672943
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,1,1,128,1,float16,float16,0,0.026791998744010927
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,1,1,128,1,float16,fp8,0,0.027508801221847533
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1536,1,1,128,1,fp8,fp8,0,0.027643200755119324
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,1,1,128,1,float16,float16,0,0.020739200711250304
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,1,1,128,1,fp8,fp8,0,0.020750400424003602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,1,1,128,1,float16,fp8,0,0.01974720060825348
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,1,1,128,1,fp8,fp8,0,0.019966399669647215
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,1,1,128,1,float16,float16,0,0.08876000046730041
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,2048,1,1,128,1,float16,float16,0,0.07364320158958435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,1,1,128,1,float16,fp8,0,0.0911903977394104
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1024,1,1,128,1,fp8,fp8,0,0.09150720238685608
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,1,1,128,1,float16,float16,0,0.053865599632263186
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,1,1,128,1,float16,fp8,0,0.05374079942703247
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1024,1,1,128,1,fp8,fp8,0,0.05360320210456848
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,8192,1,1,128,1,float16,fp8,0,0.07110239863395691
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,1,1,128,1,float16,float16,0,0.0316864013671875
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,1,1,128,1,float16,fp8,0,0.032369598746299744
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1024,1,1,128,1,fp8,fp8,0,0.03249120116233826
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,1,1,128,1,float16,fp8,0,0.028942400217056276
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,1,1,128,1,fp8,fp8,0,0.02868480086326599
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,1,1,128,1,float16,float16,0,0.02038400024175644
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,1,1,128,1,float16,float16,0,0.0217071995139122
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,1,1,128,1,float16,fp8,0,0.02162880003452301
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1024,1,1,128,1,fp8,fp8,0,0.021747200191020964
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,1,1,128,1,float16,float16,0,0.01847040057182312
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,2048,1,1,128,1,fp8,fp8,0,0.020102399587631225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,1,1,128,1,float16,fp8,0,0.019651199877262115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1024,1,1,128,1,fp8,fp8,0,0.019776000082492827
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,1,1,128,1,float16,float16,0,0.017771199345588684
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,1,1,128,1,float16,fp8,0,0.018855999410152435
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,1,1,128,1,float16,float16,0,0.0664896011352539
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,1,1,128,1,float16,fp8,0,0.06871520280838013
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,3072,1,1,128,1,float16,fp8,0,0.05335680246353149
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,512,1,1,128,1,fp8,fp8,0,0.06907039880752563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,1,1,128,1,float16,float16,0,0.039444801211357114
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,1,1,128,1,float16,fp8,0,0.04041279852390289
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,1,1,128,1,float16,float16,0,0.023056000471115112
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,1,1,128,1,float16,fp8,0,0.02468799948692322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,512,1,1,128,1,fp8,fp8,0,0.024607999622821806
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,1,1,128,1,float16,float16,0,0.01982239931821823
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,1,1,128,1,float16,fp8,0,0.021495999395847322
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,512,1,1,128,1,fp8,fp8,0,0.021473599970340727
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,1,1,128,1,float16,float16,0,0.018355199694633485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,1,1,128,1,float16,fp8,0,0.020024000108242034
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,512,1,1,128,1,fp8,fp8,0,0.020043200254440306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,1,1,128,1,float16,float16,0,0.017102399468421937
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,1,1,128,1,fp8,fp8,0,0.019108800590038298
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,1,1,128,1,float16,float16,0,0.016606399416923524
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1536,1,1,128,1,float16,float16,0,0.018676799535751343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,1,1,128,1,float16,fp8,0,0.018636800348758698
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,512,1,1,128,1,fp8,fp8,0,0.018803200125694274
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,1,1,128,1,float16,float16,0,0.01648160070180893
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,1,1,128,1,float16,fp8,0,0.018886399269104005
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,512,1,1,128,1,fp8,fp8,0,0.018622399866580965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,1,1,128,1,float16,float16,0,0.03491039872169495
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,1,1,128,1,float16,fp8,0,0.03518880009651184
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,256,1,1,128,1,fp8,fp8,0,0.0353632003068924
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,1,1,128,1,float16,fp8,0,0.021537600457668303
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,1,1,128,1,fp8,fp8,0,0.021480000019073485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,1,1,128,1,float16,float16,0,0.017284800112247468
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,1,1,128,1,float16,fp8,0,0.018251200020313264
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,256,1,1,128,1,fp8,fp8,0,0.018177600204944612
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,1,1,128,1,float16,float16,0,0.015081599354743958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1024,1,1,128,1,float16,float16,0,0.02729920148849487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,1,1,128,1,float16,fp8,0,0.01653279960155487
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,256,1,1,128,1,fp8,fp8,0,0.016492800414562227
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,1,1,128,1,float16,float16,0,0.014286400377750396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,1,1,128,1,float16,fp8,0,0.015614399313926696
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,256,1,1,128,1,fp8,fp8,0,0.015380799770355225
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,1,1,128,1,float16,float16,0,0.013622400164604188
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,1,1,128,1,float16,fp8,0,0.014955200254917145
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,256,1,1,128,1,fp8,fp8,0,0.014923200011253357
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,1,1,128,1,float16,float16,0,0.013729600608348847
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,1,1,128,1,float16,fp8,0,0.014903999865055084
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1024,1,1,128,1,fp8,fp8,0,0.018955199420452117
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,1,1,128,1,float16,float16,0,0.013555200397968292
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,1,1,128,1,float16,fp8,0,0.014662399888038635
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,256,1,1,128,1,fp8,fp8,0,0.014724799990653991
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,1,1,128,1,float16,float16,0,0.02093919962644577
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,1,1,128,1,float16,fp8,0,0.020955200493335723
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,128,1,1,128,1,fp8,fp8,0,0.021028800308704375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,512,1,1,128,1,fp8,fp8,0,0.04040960073471069
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,1,1,128,1,float16,fp8,0,0.01650400012731552
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,1,1,128,1,fp8,fp8,0,0.016758400201797485
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,1,1,128,1,float16,float16,0,0.014345599710941315
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,1,1,128,1,float16,fp8,0,0.014454400539398194
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,1,1,128,1,float16,float16,0,0.012542399764060973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,1,1,128,1,float16,fp8,0,0.013291199505329133
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,512,1,1,128,1,float16,fp8,0,0.019247999787330626
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,128,1,1,128,1,fp8,fp8,0,0.01334560066461563
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1536,1,1,128,1,float16,fp8,0,0.020980800688266753
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,1,1,128,1,float16,fp8,0,0.013030399382114411
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,1,1,128,1,float16,float16,0,0.012124799937009812
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,1,1,128,1,fp8,fp8,0,0.012999999523162841
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,1,1,128,1,float16,fp8,0,0.012804800271987915
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,128,1,1,128,1,fp8,fp8,0,0.01265760064125061
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,1,1,128,1,float16,float16,0,0.011803200095891952
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,1,1,128,1,float16,fp8,0,0.012457600235939026
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,128,1,1,128,1,fp8,fp8,0,0.012316799908876418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,1,1,128,1,float16,float16,0,0.011816000193357467
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,1,1,128,1,float16,fp8,0,0.012427199631929398
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,128,1,1,128,1,fp8,fp8,0,0.01247360035777092
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,1,1,128,1,float16,float16,0,0.017441600561141968
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,256,1,1,128,1,float16,float16,0,0.021270400285720824
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,1,1,128,1,float16,fp8,0,0.017846399545669557
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,64,1,1,128,1,fp8,fp8,0,0.018199999630451203
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,1,1,128,1,float16,float16,0,0.014771200716495514
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,1,1,128,1,fp8,fp8,0,0.015094399452209473
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,1,1,128,1,float16,float16,0,0.013284799456596375
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,1,1,128,1,float16,fp8,0,0.013699199259281158
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,64,1,1,128,1,fp8,fp8,0,0.014022399485111237
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,1,1,128,1,float16,float16,0,0.012368000298738479
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,1,1,128,1,float16,fp8,0,0.013038399815559387
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,64,1,1,128,1,fp8,fp8,0,0.012742400169372559
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,1,1,128,1,float16,float16,0,0.012049599736928939
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,1,1,128,1,float16,fp8,0,0.012873600423336028
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,1,1,128,1,float16,float16,0,0.011664000153541566
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,1,1,128,1,float16,fp8,0,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,64,1,1,128,1,fp8,fp8,0,0.012347199767827988
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,1,1,128,1,float16,float16,0,0.011641599982976914
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,1,1,128,1,fp8,fp8,0,0.012494400143623352
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,1,1,128,1,float16,float16,0,0.011585599929094314
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,256,1,1,128,1,fp8,fp8,0,0.01483680009841919
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,1,1,128,1,float16,fp8,0,0.012238399684429168
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,64,1,1,128,1,fp8,fp8,0,0.012401600182056428
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,128,1,1,128,1,float16,float16,0,0.016313600540161132
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,1,1,128,1,float16,fp8,0,0.01703840047121048
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,1,1,128,1,fp8,fp8,0,0.01706079989671707
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,128,1,1,128,1,fp8,fp8,0,0.014710399508476257
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,1,1,128,1,float16,float16,0,0.013785600662231445
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,1,1,128,1,float16,fp8,0,0.01467359960079193
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,32,1,1,128,1,fp8,fp8,0,0.014796799421310425
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,1,1,128,1,float16,float16,0,0.012625600397586822
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,1,1,128,1,fp8,fp8,0,0.013316799700260163
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,128,1,1,128,1,float16,float16,0,0.012432000041007996
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,1,1,128,1,float16,float16,0,0.011763200163841248
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,1,1,128,1,float16,fp8,0,0.012899200618267059
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,32,1,1,128,1,fp8,fp8,0,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,1,1,128,1,float16,float16,0,0.011608000099658965
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,1,1,128,1,float16,fp8,0,0.012641599774360657
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,32,1,1,128,1,fp8,fp8,0,0.012342400103807449
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,1,1,128,1,float16,float16,0,0.011451199650764465
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,1,1,128,1,float16,fp8,0,0.01228799968957901
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,32,1,1,128,1,fp8,fp8,0,0.012358400225639343
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,1,1,128,1,float16,float16,0,0.011449600011110306
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,1,1,128,1,float16,fp8,0,0.012297599762678146
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,32,1,1,128,1,fp8,fp8,0,0.012377600371837615
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,1,1,128,1,float16,float16,0,0.011456000059843064
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,1,1,128,1,float16,fp8,0,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,32,1,1,128,1,fp8,fp8,0,0.012375999987125397
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,64,1,1,128,1,float16,fp8,0,0.01539359986782074
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,1,1,128,1,float16,fp8,0,0.016323199868202208
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,1,1,128,1,float16,float16,0,0.013556799292564392
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,1,1,128,1,float16,fp8,0,0.014212800562381745
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,16,1,1,128,1,fp8,fp8,0,0.014180800318717957
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,64,1,1,128,1,fp8,fp8,0,0.012566399574279786
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,1,1,128,1,float16,float16,0,0.012521600723266602
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,1,1,128,1,float16,fp8,0,0.013329599797725678
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,16,1,1,128,1,fp8,fp8,0,0.013368000090122224
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,1,1,128,1,float16,float16,0,0.011900799721479416
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,1,1,128,1,fp8,fp8,0,0.012724800407886505
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,1,1,128,1,float16,float16,0,0.011777599900960922
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,1,1,128,1,float16,fp8,0,0.012513600289821625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,16,1,1,128,1,fp8,fp8,0,0.012567999958992004
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,1,1,128,1,float16,float16,0,0.0115167997777462
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,1,1,128,1,float16,fp8,0,0.012220799922943115
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,16,1,1,128,1,fp8,fp8,0,0.012223999947309494
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,1,1,128,1,float16,float16,0,0.011556799709796905
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,1,1,128,1,float16,fp8,0,0.012305600196123123
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,32,1,1,128,1,float16,float16,0,0.015787200629711153
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,1,1,128,1,float16,float16,0,0.011627200245857238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,1,1,128,1,float16,fp8,0,0.012193600088357926
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,16,1,1,128,1,fp8,fp8,0,0.01220960021018982
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,1,1,128,1,float16,float16,0,0.014854399859905243
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,1,1,128,1,float16,fp8,0,0.015857599675655365
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,1,1,128,1,float16,float16,0,0.013152000308036805
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,32,1,1,128,1,float16,fp8,0,0.013347199559211731
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,1,1,128,1,float16,fp8,0,0.01414559930562973
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,64,1,1,1,128,1,fp8,fp8,0,0.014204800128936768
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,1,1,128,1,float16,fp8,0,0.01311360001564026
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,1,1,128,1,fp8,fp8,0,0.013089600205421447
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,1,1,128,1,float16,float16,0,0.011720000207424164
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,1,1,128,1,float16,fp8,0,0.012425599992275238
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,1,1,1,128,1,fp8,fp8,0,0.01255040019750595
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,1,1,128,1,float16,float16,0,0.011683200299739838
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,1,1,128,1,float16,fp8,0,0.012539200484752655
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,1,1,128,1,float16,float16,0,0.015081599354743958
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,1,1,128,1,float16,float16,0,0.011393599957227708
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,16,1,1,128,1,fp8,fp8,0,0.016238400340080263
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,1,1,128,1,float16,fp8,0,0.012057600170373916
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,4,1,1,1,128,1,fp8,fp8,0,0.01226240023970604
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,1,1,128,1,float16,float16,0,0.011448000371456147
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,1,1,128,1,float16,fp8,0,0.012145599722862244
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,1,1,1,128,1,fp8,fp8,0,0.012265600264072418
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,1,1,128,1,float16,float16,0,0.011379200220108032
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,1,1,128,1,fp8,fp8,0,0.012031999975442886
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,16,1,1,128,1,fp8,fp8,0,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,128,1,1,1,128,1,fp8,fp8,0,0.015812799334526062
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,32,1,1,1,128,1,float16,float16,0,0.012723200023174286
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,8,1,1,1,128,1,fp8,fp8,0,0.012513600289821625
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,1,1,1,1,128,1,float16,fp8,0,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,2,64,1,1,128,1,float16,fp8,0,0.01223680004477501
SGLang,0.5.9,NVIDIA H200,context_attention,flash_attention,16,16,1,1,128,1,float16,fp8,0,0.012676799297332763
