framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,1,0.045363199710845944
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,7,0.04524640142917633
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,3,0.04528799951076508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,15,0.04521119892597199
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,1,0.03703039884567261
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,31,0.0454479992389679
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,31,0.03702400028705597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,7,0.037088000774383546
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,63,0.049327999353408813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,127,0.05568959712982178
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,15,0.037067198753356935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,127,0.03919360041618347
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,255,0.0864799976348877
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,255,0.04842720031738281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,511,0.08527519702911376
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,1023,0.30072000026702883
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,1023,0.16072800159454345
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,3,0.037089601159095764
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,2047,0.5904928207397461
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,63,0.03703039884567261
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,2047,0.3116368055343628
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,1,0.010529600083827972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,3,0.01063840016722679
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,511,0.15686880350112914
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,7,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,15,0.01055999994277954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,1,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,31,0.010486400127410889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,3,0.008491200208663941
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,63,0.010593599826097488
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,31,0.00907519981265068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,63,0.008910399675369263
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,127,0.010523200035095215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,15,0.010222399979829789
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,127,0.009459199756383896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,511,0.010513599961996078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,255,0.010489600151777268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,511,0.010318399965763092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,1023,0.012494400143623352
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,1023,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,2047,0.011343999952077865
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,2047,0.014542399346828461
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,1,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,3,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,3,0.008321599662303924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,7,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,7,0.008451200276613235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,15,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,15,0.008302400261163712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,31,0.010315199941396713
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,31,0.008497600257396699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,63,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,63,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,127,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,127,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,255,0.01061440035700798
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,255,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,511,0.012408000230789185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,511,0.00851999968290329
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,1023,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,1023,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,7,0.00841279998421669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,2047,0.03112800121307373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,2047,0.012588800489902496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,1,0.02694559991359711
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,1,0.02268960028886795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,3,0.026924800872802735
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,7,0.02690559923648834
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,7,0.022595199942588805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,15,0.0269679993391037
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,1,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,15,0.022728000581264497
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,31,0.022646400332450866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,31,0.026899200677871705
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,63,0.026875200867652892
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,127,0.03352000117301941
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,127,0.022705599665641785
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,255,0.049980801343917844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,255,0.029262399673461913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,511,0.08583040237426758
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,511,0.047856000065803525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,1023,0.15800479650497437
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,1023,0.0855296015739441
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,2047,0.3001552104949951
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,255,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,2047,0.16197919845581055
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,1,0.012454400211572647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,3,0.02271520048379898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,1,0.010320000350475311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,3,0.012329600006341934
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,7,0.012377600371837615
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,3,0.009607999771833419
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,15,0.012164799869060517
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,15,0.00963039994239807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,31,0.009814400225877762
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,63,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,63,0.02268799990415573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,63,0.008406399935483932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,127,0.0110384002327919
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,127,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,255,0.010467199981212616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,255,0.00987040027976036
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,511,0.01454080045223236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,511,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,1023,0.03028320074081421
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,1023,0.012577599287033081
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,2047,0.04939039945602417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,2047,0.02786239981651306
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,1,0.08015679717063903
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,1,0.06583359837532043
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,3,0.08014240264892578
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,7,0.010284800082445145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,3,0.06587679982185364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,31,0.011800000071525573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,7,0.06577759981155396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,7,0.08015040159225464
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,15,0.08221279978752136
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,15,0.0678991973400116
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,31,0.08425120115280152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,31,0.06754400134086609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,63,0.08433600068092346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,63,0.06785439848899841
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,127,0.094404798746109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,127,0.06832320094108582
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,255,0.08663840293884277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,255,0.15877599716186525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,511,0.3018079996109009
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,511,0.15918879508972167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,1,0.1499343991279602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,1023,0.5872496128082275
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,1023,0.3110192060470581
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,3,0.12518080472946166
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,3,0.15066560506820678
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,7,0.15417439937591554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,7,0.12640800476074218
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,15,0.15399839878082275
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,15,0.1272447943687439
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,31,0.15466400384902954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,31,0.127183997631073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,63,0.15416159629821777
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,63,0.12726720571517944
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,127,0.17669440507888795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,127,0.12795840501785277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,255,0.30467839241027833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,255,0.16401760578155516
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,1,0.2440000057220459
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,3,0.29547359943389895
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,1,0.12518240213394166
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,7,0.29534239768981935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,7,0.24524319171905518
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,15,0.29531519412994384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,15,0.24489281177520753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,31,0.24412961006164552
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,63,0.2965359926223755
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,63,0.2441119909286499
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,127,0.3486543893814087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,1,0.29324800968170167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,127,0.24639201164245605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,1,0.013596799969673157
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,1,0.011107199639081956
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,3,0.014004799723625182
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,3,0.24404480457305908
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,3,0.01106560006737709
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,7,0.011529599875211715
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,15,0.014406399428844452
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,15,0.011635199934244157
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,31,0.014459200203418732
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,31,0.011537600308656693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,63,0.014494399726390838
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,63,0.011497599631547928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,127,0.014416000247001648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,127,0.012375999987125397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,255,0.012439999729394913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,511,0.032734400033950804
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,511,0.014620800316333771
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,31,0.29556319713592527
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,1023,0.02908639907836914
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,2047,0.08733599781990051
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,2047,0.04740639925003052
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,1,0.5796592235565186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,1,0.47993760108947753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,7,0.014318400621414184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,3,0.5788959980010986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,3,0.48145599365234376
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,255,0.01451839953660965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,7,0.5785088062286377
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,7,0.4813663959503174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,1023,0.05052800178527832
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,15,0.5794928073883057
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,15,0.4804992198944092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,31,0.48140640258789064
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,63,0.580292797088623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,63,0.4797359943389893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,1,1.1533727645874023
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,1,0.9541520118713379
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,3,1.150984001159668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,3,0.9551695823669434
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,7,0.9538047790527344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,15,1.1519248008728027
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,31,0.5796319961547851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,15,0.9537152290344239
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,31,1.1529199600219726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,1,0.018545599281787874
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,1,0.014696000516414643
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,3,0.018611200153827667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,3,0.014630399644374847
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,7,0.018596799671649934
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,7,0.014574399590492249
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,15,0.018603199720382692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,15,0.014672000706195832
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,7,1.1513104438781738
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,31,0.01857919991016388
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,31,0.01459839940071106
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,63,0.01866080015897751
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,127,0.018510399758815764
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,127,0.01467359960079193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,255,0.03083840012550354
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,255,0.01464959979057312
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,511,0.04989440143108368
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,511,0.028977599740028382
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,1023,0.08572480082511902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,1023,0.0477616012096405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,2047,0.15705280303955077
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,31,0.9522192001342773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,2047,0.08564000129699707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,1,0.03513120114803314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,1,0.03089439868927002
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,3,0.035076799988746646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,3,0.030921599268913268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,7,0.03508960008621216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,7,0.03094080090522766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,15,0.03500480055809021
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,15,0.030142399668693542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,31,0.034985598921775815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,31,0.029974400997161865
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,63,0.03707360029220581
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,63,0.030500799417495728
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,127,0.03101919889450073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,127,0.045270401239395144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,63,0.014692799746990204
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,255,0.06718080043792725
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,255,0.039211198687553406
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,511,0.06594880223274231
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,1023,0.22758080959320068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,1023,0.12344800233840943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,2047,0.44638562202453613
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,1,0.010507199913263321
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,2047,0.2360703945159912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,3,0.010502400249242783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,7,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,511,0.12108160257339477
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,15,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,7,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,3,0.009808000177145004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,1,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,31,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,31,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,63,0.012196800112724305
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,63,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,127,0.010502400249242783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,127,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,255,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,255,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,511,0.012465599924325943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,511,0.010502400249242783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,1023,0.012508800625801087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,1023,0.01058880016207695
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,2047,0.014497600495815277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,2047,0.012537600100040435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,1,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,1,0.008419200032949447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,3,0.008484800159931184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,3,0.008428800106048583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,7,0.00841120034456253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,7,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,15,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,15,0.008417599648237229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,31,0.008452799916267396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,31,0.00835999995470047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,63,0.008481600135564805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,63,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,127,0.008398400247097015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,255,0.010326399654150008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,255,0.008414400368928909
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,511,0.010499200224876404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,511,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,1023,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,2047,0.018651199340820313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,2047,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,1,0.022678400576114654
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,1,0.01865759938955307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,3,0.022787199914455415
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,3,0.018651199340820313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,7,0.022782400250434875
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,15,0.010342399775981902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,15,0.022697600722312927
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,15,0.01879040002822876
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,31,0.022761599719524385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,31,0.018848000466823576
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,63,0.02269600033760071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,127,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,63,0.018824000656604768
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,127,0.02582559883594513
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,127,0.018777599930763243
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,255,0.04065600037574768
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,255,0.020691199600696562
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,511,0.06765120029449463
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,511,0.03875199854373932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,1023,0.12165759801864624
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,7,0.01865279972553253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,1023,0.06687039732933045
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,2047,0.1233456015586853
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,2047,0.23106238842010499
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,1,0.01051200032234192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,1,0.008548799902200699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,3,0.010520000010728836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,3,0.008430399745702744
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,7,0.01053759977221489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,7,0.008444800227880477
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,15,0.010516799986362457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,15,0.008436799794435502
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,31,0.01058719977736473
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,31,0.00835679993033409
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,63,0.010585600137710571
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,63,0.008430399745702744
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,127,0.01058880016207695
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,127,0.008486399799585343
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,255,0.010683199763298035
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,511,0.012708799540996551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,511,0.010595200210809707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,1023,0.020771199464797975
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,1023,0.012532800436019897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,2047,0.044552001357078555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,1023,0.012521600723266602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,2047,0.01881760060787201
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,1,0.061740797758102414
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,1,0.051534402370452884
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,3,0.05144000053405762
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,3,0.06168320178985596
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,7,0.06172800064086914
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,7,0.05145760178565979
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,15,0.05140799880027771
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,31,0.06569920182228088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,31,0.05154079794883728
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,63,0.06573920249938965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,63,0.05239840149879456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,127,0.07406240105628967
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,255,0.12124799489974976
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,255,0.06835200190544129
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,255,0.008484800159931184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,511,0.22808480262756348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,511,0.12273759841918945
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,1023,0.4430367946624756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,1023,0.2362207889556885
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,1,0.11490720510482788
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,1,0.0964784026145935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,3,0.11488800048828125
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,3,0.09647520184516907
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,7,0.11532319784164428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,7,0.09642080068588257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,15,0.11709920167922974
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,15,0.09644320011138915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,31,0.09642239809036254
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,31,0.1171023964881897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,63,0.11792000532150268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,63,0.09806079864501953
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,127,0.09866719841957092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,127,0.053947198390960696
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,255,0.23024160861968995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,255,0.12745920419692994
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,1,0.22133278846740723
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,1,0.184934401512146
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,3,0.22347679138183593
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,3,0.18486720323562622
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,7,0.18471360206604004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,15,0.2239680051803589
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,15,0.061680001020431516
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,15,0.18662240505218505
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,31,0.22416160106658936
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,31,0.18572959899902344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,63,0.2250976085662842
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,63,0.18592159748077391
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,127,0.13415520191192626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,127,0.19043840169906617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,1,0.01241919994354248
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,1,0.010542400181293488
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,3,0.012564800679683685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,3,0.010601600259542465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,7,0.012470400333404541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,7,0.010480000078678131
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,15,0.012449599802494049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,15,0.01055999994277954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,31,0.0124719999730587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,31,0.010532800108194351
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,63,0.012481600046157837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,63,0.010569600015878677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,127,0.012441600114107132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,127,0.010366400331258773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,255,0.012444800138473511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,255,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,7,0.2236543893814087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,511,0.012535999715328216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,1023,0.04001759886741638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,1023,0.01820160001516342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,2047,0.06981920003890991
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,2047,0.03814719915390015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,1,0.4363232135772705
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,1,0.3627216100692749
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,127,0.263592004776001
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,3,0.4363999843597412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,3,0.3631743907928467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,7,0.4366752147674561
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,7,0.3645983934402466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,511,0.020468799769878386
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,15,0.43653278350830077
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,15,0.36410400867462156
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,31,0.4364304065704346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,31,0.36269440650939944
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,63,0.43822240829467773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,63,0.36261920928955077
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,1,0.8647551536560059
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,1,0.7191919803619384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,3,0.8657471656799316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,3,0.7182688236236572
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,7,0.8660752296447753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,7,0.7176784038543701
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,15,0.8647919654846191
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,1,0.016484799981117248
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,1,0.012547199428081513
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,15,0.7172239780426025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,3,0.016622400283813475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,31,0.7168303966522217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,7,0.016548800468444824
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,7,0.012579199671745301
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,15,0.016579200327396394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,15,0.012479999661445617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,31,0.01658879965543747
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,31,0.012567999958992004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,63,0.016547200083732606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,63,0.013355199992656708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,127,0.015636800229549407
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,127,0.01361120045185089
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,255,0.018782399594783783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,255,0.012590399384498597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,511,0.04025920033454895
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,511,0.018603199720382692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,1023,0.0691968023777008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,1023,0.038971200585365295
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,2047,0.1254688024520874
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,2047,0.06693440079689025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,1,0.030955201387405394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,3,0.030955201387405394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,31,0.8666192054748535
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,3,0.01449279934167862
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,7,0.031012800335884095
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,15,0.030964800715446474
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,1,0.024899199604988098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,31,0.03089280128479004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,3,0.02484479993581772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,31,0.0267551988363266
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,63,0.03267360031604767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,63,0.026800000667572023
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,127,0.03840959966182709
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,127,0.02677760124206543
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,255,0.05752800107002258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,255,0.034215998649597165
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,511,0.10257279872894287
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,511,0.05672640204429626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,1023,0.19216480255126953
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,1023,0.10456160306930543
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,7,0.026748800277709962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,2047,0.19941920042037964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,2047,0.3718559980392456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,1,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,3,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,15,0.02683520019054413
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,7,0.010499200224876404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,15,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,3,0.008560000360012055
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,7,0.008561599999666214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,1,0.008497600257396699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,31,0.010358399897813796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,31,0.008561599999666214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,63,0.008561599999666214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,127,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,127,0.008452799916267396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,255,0.010577599704265594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,255,0.008721599727869034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,511,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,511,0.008497600257396699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,1023,0.011751999706029892
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,1023,0.010289599746465683
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,2047,0.01252799928188324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,2047,0.010503999888896942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,1,0.008668799698352814
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,1,0.00835840031504631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,3,0.008542399853467941
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,3,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,7,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,7,0.008363199979066848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,15,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,15,0.008481600135564805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,31,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,31,0.008390399813652038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,63,0.0086496002972126
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,63,0.008287999778985977
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,127,0.009247999638319016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,127,0.008396799862384795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,255,0.009440000355243682
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,255,0.008259200304746629
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,511,0.010539200156927109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,511,0.008585599809885025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,1023,0.012641599774360657
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,1023,0.010355199873447418
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,2047,0.016663999855518342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,2047,0.012511999905109405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,63,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,1,0.020640000700950623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,1,0.016596800088882445
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,3,0.020571200549602507
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,3,0.016523200273513793
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,7,0.020497600734233856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,7,0.016603200137615202
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,15,0.020572799444198608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,15,0.016641600430011748
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,31,0.020571200549602507
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,31,0.016579200327396394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,63,0.020606400072574617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,127,0.020857599377632142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,127,0.01669919937849045
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,255,0.035062399506568906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,255,0.018294399976730345
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,511,0.05776320099830627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,511,0.033670398592948916
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,1023,0.10252159833908081
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,1023,0.05706080198287964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,2047,0.19209760427474976
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,2047,0.1045024037361145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,1,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,1,0.00852160006761551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,3,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,3,0.008643200248479843
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,15,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,7,0.010547199845314026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,7,0.008396799862384795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,15,0.010542400181293488
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,15,0.008367999643087386
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,31,0.010576000064611435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,31,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,63,0.008892799913883209
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,127,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,127,0.009385599941015243
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,255,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,63,0.016627199947834015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,255,0.009196799993515015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,511,0.012494400143623352
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,511,0.0104592002928257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,1023,0.01252799928188324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,2047,0.04118080139160156
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,2047,0.0183119997382164
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,1,0.05346239805221557
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,3,0.053508800268173215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,3,0.04435519874095917
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,7,0.053491199016571046
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,7,0.04423199892044068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,15,0.053390401601791385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,15,0.04424479901790619
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,31,0.05544319748878479
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,31,0.045219200849533084
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,63,0.056201601028442384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,63,0.0452208012342453
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,127,0.06328319907188415
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,127,0.045484799146652224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,255,0.10262080430984497
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,255,0.059087997674942015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,63,0.010790400207042694
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,511,0.1921712040901184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,1023,0.01666879951953888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,1,0.04447839856147766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,1023,0.37094080448150635
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,1023,0.19828959703445434
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,1,0.09646559953689575
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,3,0.08215519785881042
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,3,0.09646559953689575
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,7,0.0985040009021759
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,7,0.08219040036201478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,15,0.1005552053451538
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,15,0.08207200169563293
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,31,0.10055840015411377
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,63,0.10057439804077148
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,63,0.08215680122375488
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,127,0.11647200584411621
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,127,0.08323040008544921
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,255,0.19311840534210206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,255,0.10824639797210693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,1,0.18483999967575074
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,1,0.15581599473953248
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,511,0.10373920202255249
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,3,0.18650399446487426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,3,0.1556815981864929
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,7,0.18855999708175658
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,7,0.15593600273132324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,15,0.18860479593276977
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,15,0.15595840215682982
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,31,0.08230559825897217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,31,0.18837920427322388
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,31,0.15596799850463866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,63,0.18861119747161864
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,63,0.1559775948524475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,127,0.21529760360717773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,1,0.012057600170373916
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,1,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,3,0.012374400347471236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,3,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,7,0.012470400333404541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,7,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,15,0.012387199699878693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,15,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,31,0.010596799850463866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,31,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,63,0.010635200142860412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,63,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,127,0.010745599865913391
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,1,0.08205119967460632
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,127,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,255,0.0124719999730587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,255,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,511,0.014819200336933135
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,511,0.012478400021791458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,1023,0.03702400028705597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,1023,0.01653600037097931
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,2047,0.06296799778938293
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,1,0.3649600028991699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,127,0.1581104040145874
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,3,0.36562559604644773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,3,0.3034480094909668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,7,0.3650480031967163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,7,0.3045775890350342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,15,0.36512959003448486
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,15,0.3035952091217041
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,31,0.3656080007553101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,31,0.3036335945129395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,63,0.366809606552124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,2047,0.035812801122665404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,63,0.30374879837036134
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,1,0.6002480030059815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,1,0.7228240013122559
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,3,0.7220240116119385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,3,0.6005136013031006
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,7,0.7231567859649658
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,7,0.6000207901000977
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,15,0.7222335815429688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,15,0.6000288009643555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,1,0.014577600359916686
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,31,0.723145580291748
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,1,0.012611199915409089
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,3,0.01257600039243698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,3,0.014870400726795196
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,7,0.014641599357128143
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,7,0.012641599774360657
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,31,0.5985536098480224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,15,0.014664000272750855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,31,0.01456640064716339
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,31,0.012521600723266602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,63,0.014574399590492249
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,63,0.012615999579429627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,127,0.014563199877738953
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,127,0.012625600397586822
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,255,0.014563199877738953
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,255,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,511,0.035132798552513125
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,511,0.01656319946050644
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,1023,0.058254402875900266
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,1023,0.03459199965000152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,2047,0.10518720149993896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,1,0.026849600672721862
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,1,0.3033247947692871
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,1,0.02265920042991638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,3,0.026737600564956665
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,3,0.022716799378395082
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,7,0.02671839892864227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,7,0.02272160053253174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,15,0.02688480019569397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,15,0.022707200050354003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,31,0.026820799708366393
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,31,0.0227183997631073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,63,0.026814401149749756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,63,0.02263839989900589
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,127,0.03157280087471008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,127,0.02272319942712784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,255,0.04807040095329285
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,255,0.028974398970603943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,511,0.08318719863891602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,1023,0.15560959577560424
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,1023,0.08582879900932312
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,15,0.01265760064125061
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,2047,0.2992944002151489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,2047,0.16255199909210205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,2047,0.057815998792648315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,1,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,4095,0.3140719890594482
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,4095,0.5882976055145264
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,3,0.010375999659299851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,511,0.047660800814628604
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,7,0.009537599980831146
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,1,0.008534400165081025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,3,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,15,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,31,0.010375999659299851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,31,0.008462399989366532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,63,0.0104592002928257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,63,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,127,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,127,0.008544000238180161
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,255,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,255,0.008566399663686752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,7,0.008428800106048583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,511,0.010356800258159637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,511,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,1023,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,1023,0.008489599823951722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,2047,0.011603199690580369
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,2047,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,4095,0.012545600533485413
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,1,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,1,0.0083856001496315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,3,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,3,0.008532799780368805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,7,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,7,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,15,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,15,0.010281600058078766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,31,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,31,0.010520000010728836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,63,0.01053600013256073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,127,0.010564800351858139
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,127,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,63,0.008564800024032593
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,255,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,255,0.00843999981880188
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,511,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,511,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,1023,0.010627199709415436
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,2047,0.012595200538635254
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,1023,0.010592000186443329
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,2047,0.011707200109958649
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,4095,0.029262399673461913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,4095,0.014508800208568573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,1,0.018273599445819855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,1,0.014679999649524688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,3,0.016774399578571318
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,3,0.014591999351978302
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,7,0.01680160015821457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,7,0.014608000218868256
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,15,0.01661760061979294
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,15,0.014641599357128143
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,31,0.016659200191497803
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,31,0.014577600359916686
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,63,0.01849759966135025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,63,0.014611199498176575
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,127,0.01849759966135025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,4095,0.010545600205659866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,255,0.029151999950408937
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,511,0.04755200147628784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,511,0.0288783997297287
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,1023,0.0837984025478363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,1023,0.04801119863986969
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,2047,0.15565919876098633
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,2047,0.08599680066108703
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,15,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,4095,0.2997648000717163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,4095,0.16270560026168823
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,1,0.008508799970149994
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,1,0.008347199857234954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,3,0.008508799970149994
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,3,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,7,0.008673600107431411
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,7,0.00841279998421669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,15,0.008868800103664398
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,15,0.008318399637937545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,31,0.009244800359010697
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,31,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,63,0.00968639999628067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,63,0.008312000334262848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,127,0.010286399722099304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,127,0.008348800241947174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,255,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,255,0.008352000266313553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,511,0.010513599961996078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,511,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,127,0.014499199390411378
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,1023,0.012617599964141846
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,1023,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,2047,0.028964799642562867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,2047,0.012547199428081513
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,4095,0.046619200706481935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,1,0.0448415994644165
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,1,0.03707199990749359
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,3,0.044099199771881106
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,3,0.03704319894313812
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,255,0.01459839940071106
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,4095,0.027459201216697694
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,7,0.03706879913806915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,15,0.04330720007419586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,31,0.04522239863872528
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,7,0.04445759952068329
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,31,0.03720000088214874
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,63,0.04655359983444214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,63,0.03718560039997101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,127,0.05172640085220337
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,127,0.039166399836540224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,255,0.08454239964485169
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,255,0.05049279928207397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,511,0.08540800213813782
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,511,0.1558176040649414
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,1023,0.16080479621887206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,1023,0.2996975898742676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,2047,0.31309919357299804
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,2047,0.5862959861755371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,1,0.07994719743728637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,1,0.06772480010986329
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,3,0.07992640137672424
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,3,0.06775680184364319
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,7,0.06703199744224549
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,7,0.07984640002250672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,15,0.082150399684906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,15,0.06692000031471253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,15,0.037143999338150026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,31,0.08211519718170165
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,31,0.06774240136146545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,63,0.06785280108451844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,127,0.09492160081863403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,127,0.06871359944343566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,255,0.15654560327529907
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,255,0.08980000019073486
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,1,0.14974080324172973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,1,0.125273597240448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,3,0.14976799488067627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,3,0.12526400089263917
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,7,0.15176960229873657
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,7,0.1255903959274292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,15,0.15185600519180298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,15,0.12701120376586914
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,31,0.15199040174484252
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,31,0.1272096037864685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,63,0.15198880434036255
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,63,0.08223680257797242
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,127,0.1783776044845581
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,127,0.12895679473876953
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,511,0.2991744041442871
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,255,0.17021119594573975
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,255,0.3031775951385498
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,1,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,1,0.01034879982471466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,3,0.010550399869680404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,3,0.010267200320959092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,7,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,15,0.010636799782514573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,15,0.009166400134563445
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,31,0.010542400181293488
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,31,0.009329599887132644
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,63,0.01053439974784851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,63,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,127,0.010572800040245056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,127,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,255,0.010582400113344192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,255,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,511,0.012649600207805634
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,511,0.010556799918413162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,1023,0.02889919877052307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,63,0.12713600397109986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,1023,0.012643200159072877
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,2047,0.04713279902935028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,2047,0.027980801463127137
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,4095,0.08342880010604858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,4095,0.04681600034236908
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,511,0.15890879631042482
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,1,0.2929775953292847
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,1,0.2439824104309082
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,3,0.24403200149536133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,3,0.29312639236450194
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,7,0.29345600605010985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,7,0.24596641063690186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,15,0.2932800054550171
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,7,0.010553599894046783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,31,0.29427518844604494
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,31,0.24401919841766356
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,63,0.2952879905700684
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,63,0.244270396232605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,127,0.3343039989471436
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,127,0.24858720302581788
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,1,0.5797840118408203
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,1,0.48132801055908203
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,3,0.5799200057983398
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,3,0.48157920837402346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,7,0.5796832084655762
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,15,0.5797008037567138
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,15,0.4814479827880859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,31,0.48125600814819336
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,15,0.24516959190368653
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,63,0.5812032222747803
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,63,0.4813983917236328
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,1,0.012515200674533844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,1,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,3,0.01255200058221817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,3,0.012439999729394913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,7,0.012561599910259246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,7,0.011779200285673141
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,15,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,15,0.011974400281906128
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,31,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,31,0.012086399644613267
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,63,0.012503999471664428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,63,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,127,0.0126351997256279
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,127,0.010673599690198899
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,31,0.5801919937133789
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,255,0.0124719999730587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,511,0.029151999950408937
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,255,0.011348800361156463
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,511,0.01451680064201355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,1023,0.04825760126113891
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,1023,0.029028800129890443
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,2047,0.08444479703903199
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,2047,0.047491198778152464
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,4095,0.08572959899902344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,4095,0.15934400558471679
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,1,0.021996800601482392
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,3,0.02208320051431656
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,1,0.0186256006360054
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,7,0.022735999524593355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,3,0.01865919977426529
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,7,0.018619200587272643
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,15,0.022457599639892578
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,15,0.018680000305175783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,31,0.02242079973220825
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,31,0.018644799292087556
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,63,0.02203039973974228
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,63,0.01863519996404648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,127,0.02475520074367523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,7,0.4828752040863037
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,127,0.018670399487018586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,255,0.03833119869232178
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,511,0.0652127981185913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,511,0.03844479918479919
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,255,0.020788800716400147
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,1023,0.06602720022201539
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,1023,0.12029759883880616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,2047,0.23006079196929932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,1,0.010497599840164185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,4095,0.4560368061065674
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,4095,0.23773601055145263
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,3,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,7,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,2047,0.12379839420318603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,1,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,15,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,15,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,31,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,31,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,3,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,7,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,63,0.01032480001449585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,127,0.010572800040245056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,127,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,255,0.010507199913263321
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,255,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,511,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,511,0.01032480001449585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,1023,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,2047,0.012727999687194824
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,2047,0.010940799862146378
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,4095,0.013425600528717042
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,4095,0.012438400089740754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,1,0.010585600137710571
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,1,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,3,0.010620799660682679
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,3,0.010326399654150008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,7,0.010566399991512298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,7,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,15,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,15,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,31,0.010576000064611435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,31,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,63,0.010468800365924836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,63,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,127,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,127,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,255,0.010532800108194351
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,255,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,511,0.010599999874830245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,511,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,1023,0.012579199671745301
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,1023,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,2047,0.01342719942331314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,2047,0.010619200021028518
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,4095,0.018775999546051025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,4095,0.012539200484752655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,1,0.01648319959640503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,1,0.012627199292182922
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,3,0.01642879992723465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,3,0.014499199390411378
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,7,0.016337600350379945
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,7,0.014483200013637542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,15,0.014616000652313232
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,15,0.01297920048236847
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,31,0.014617599546909332
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,63,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,63,0.014524799585342408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,63,0.012563200294971466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,1023,0.010520000010728836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,127,0.01451520025730133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,127,0.014528000354766845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,255,0.014524799585342408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,511,0.03939520120620728
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,511,0.018671999871730804
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,1023,0.0678704023361206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,1023,0.038577601313591
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,2047,0.12474720478057862
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,2047,0.0678160011768341
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,4095,0.2443631887435913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,4095,0.12604000568389892
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,1,0.008636800199747085
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,1,0.008363199979066848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,3,0.008684799820184708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,3,0.008252800256013871
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,7,0.00851840004324913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,31,0.013334399461746216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,7,0.008336000144481659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,15,0.008620800077915191
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,15,0.008329600095748901
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,31,0.008588799834251403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,31,0.008352000266313553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,63,0.008531200140714646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,63,0.008312000334262848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,255,0.017688000202178956
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,127,0.008302400261163712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,255,0.008396799862384795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,511,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,511,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,1023,0.01136159971356392
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,1023,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,2047,0.018086400628089905
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,2047,0.012436799705028534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,4095,0.04163039922714233
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,4095,0.018580800294876097
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,1,0.030907198786735535
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,3,0.03497920036315918
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,3,0.030823999643325807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,7,0.03502880036830902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,7,0.028886398673057555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,15,0.034964799880981445
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,15,0.028923198580741882
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,31,0.03500159978866577
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,31,0.02892639935016632
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,63,0.03698399960994721
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,63,0.028863999247550964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,127,0.04311839938163757
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,127,0.030902400612831116
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,255,0.06584479808807372
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,127,0.008446399867534638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,255,0.039108800888061526
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,255,0.008259200304746629
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,511,0.0665727972984314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,511,0.11903680562973022
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,1023,0.1231727957725525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,1023,0.22783999443054198
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,1,0.06165120005607605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,2047,0.4437727928161621
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,1,0.05146399736404419
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,2047,0.23729920387268066
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,3,0.05154719948768616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,3,0.061654400825500486
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,7,0.061577600240707395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,1,0.034974399209022525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,15,0.061686402559280394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,31,0.06381279826164246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,31,0.05236160159111023
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,63,0.06365600228309631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,63,0.05345600247383118
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,127,0.05555199980735779
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,255,0.12111999988555908
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,255,0.06860960125923157
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,511,0.22722880840301513
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,511,0.12206399440765381
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,1,0.1149440050125122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,1,0.09649760127067566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,3,0.11486400365829467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,3,0.09650239944458008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,7,0.11519360542297363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,7,0.09654399752616882
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,15,0.1170799970626831
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,7,0.05149120092391968
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,15,0.09645599722862244
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,15,0.05134879946708679
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,31,0.1168511986732483
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,31,0.09804800152778625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,63,0.11692960262298584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,63,0.09851040244102478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,127,0.13384480476379396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,127,0.10054880380630493
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,127,0.07224479913711548
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,1,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,1,0.0084927998483181
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,3,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,3,0.008470399677753449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,7,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,7,0.008423999696969987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,15,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,15,0.008472000062465668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,31,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,31,0.008486399799585343
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,63,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,63,0.008406399935483932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,127,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,127,0.008433599770069123
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,255,0.0105103999376297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,255,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,511,0.012513600289821625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,511,0.010468800365924836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,1023,0.0208064004778862
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,1023,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,2047,0.041203200817108154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,2047,0.018603199720382692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,255,0.22947840690612792
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,4095,0.07176799774169922
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,255,0.12744640111923217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,4095,0.04178400039672851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,1,0.1845744013786316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,3,0.22169759273529052
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,3,0.18467199802398682
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,7,0.22339839935302735
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,7,0.18606239557266235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,15,0.22325439453125
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,15,0.18666399717330934
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,31,0.2232208013534546
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,31,0.1865231990814209
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,63,0.22338719367980958
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,127,0.2609647989273071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,127,0.1933583974838257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,1,0.36296160221099855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,1,0.4366464138031006
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,3,0.4364367961883545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,1,0.22129759788513184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,7,0.43645439147949217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,7,0.36401920318603515
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,15,0.43626880645751953
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,15,0.36356320381164553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,63,0.1866544008255005
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,31,0.43634557723999023
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,63,0.4373760223388672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,1,0.010604800283908844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,1,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,3,0.01074879989027977
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,3,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,7,0.010582400113344192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,7,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,15,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,15,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,31,0.012438400089740754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,31,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,63,0.012399999797344208
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,63,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,127,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,127,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,255,0.011388800293207168
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,255,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,511,0.017473599314689635
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,511,0.012470400333404541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,1023,0.03816480040550232
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,1023,0.016633599996566772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,2047,0.065583997964859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,2047,0.038878399133682254
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,4095,0.12033439874649048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,63,0.36475520133972167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,1,0.016551999747753142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,1,0.014726400375366211
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,3,0.01677280068397522
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,3,0.01464959979057312
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,7,0.016628800332546233
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,7,0.014752000570297241
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,15,0.016676799952983858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,3,0.3635535955429077
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,15,0.014603200554847717
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,31,0.016791999340057373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,31,0.014620800316333771
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,63,0.016748799383640288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,127,0.016897599399089813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,127,0.014612799882888794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,255,0.028833600878715514
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,255,0.014739200472831726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,511,0.04693120121955872
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,31,0.3628704071044922
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,1023,0.08273119926452636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,1023,0.04783360064029694
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,2047,0.1546623945236206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,2047,0.08582879900932312
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,4095,0.29886560440063475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,4095,0.16357280015945436
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,63,0.014655999839305878
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,8191,0.587827205657959
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,511,0.029078400135040282
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,1,0.010523200035095215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,8191,0.31519041061401365
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,4095,0.06635199785232544
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,1,0.010574399679899215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,7,0.01053439974784851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,7,0.010313600301742554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,3,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,15,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,3,0.008507200330495835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,31,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,31,0.008832000195980072
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,63,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,63,0.0095551997423172
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,127,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,127,0.009806399792432785
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,255,0.010486400127410889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,255,0.009547200053930283
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,511,0.010531199723482132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,511,0.009455999732017517
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,1023,0.010556799918413162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,1023,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,2047,0.012430399656295776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,2047,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,4095,0.012683199346065521
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,4095,0.012438400089740754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,8191,0.014662399888038635
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,1,0.009284800291061402
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,3,0.00931359976530075
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,1,0.008715199679136277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,3,0.008556800335645676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,7,0.008740799874067307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,7,0.008523199707269669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,15,0.009318400174379349
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,15,0.008507200330495835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,31,0.009011200070381165
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,31,0.008436799794435502
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,63,0.009145600348711013
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,127,0.010526400059461594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,127,0.008628799766302108
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,255,0.0090768001973629
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,255,0.008607999980449676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,511,0.010467199981212616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,511,0.008561599999666214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,1023,0.010555200278759003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,15,0.008499199897050858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,1023,0.010521599650382995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,2047,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,2047,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,4095,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,4095,0.011777599900960922
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,8191,0.014473600685596466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,1,0.012639999389648438
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,1,0.011147200316190719
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,3,0.012675200402736665
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,8191,0.012491200119256973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,3,0.010969600081443787
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,7,0.012513600289821625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,15,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,15,0.01183359995484352
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,31,0.012436799705028534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,31,0.010468800365924836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,63,0.012451200187206269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,63,0.01067200005054474
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,63,0.008640000224113464
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,127,0.012465599924325943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,127,0.012321600317955017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,255,0.012409599870443344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,255,0.012436799705028534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,511,0.028377598524093627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,511,0.014553600549697876
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,1023,0.04731679856777191
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,1023,0.02924480140209198
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,2047,0.08431040048599243
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,2047,0.04845919907093048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,4095,0.15830559730529786
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,4095,0.08614720106124878
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,8191,0.30593280792236327
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,1,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,8191,0.028638398647308348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,1,0.008408000320196151
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,3,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,7,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,3,0.0083856001496315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,7,0.008851200342178345
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,15,0.009414400160312652
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,15,0.008982399851083756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,31,0.009556800127029419
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,31,0.008902399986982345
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,63,0.009020800143480301
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,63,0.009180799871683121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,127,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,7,0.012503999471664428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,127,0.009401600062847137
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,255,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,255,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,511,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,511,0.009017600119113922
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,1023,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,1023,0.010574399679899215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,2047,0.011923199892044068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,4095,0.028974398970603943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,4095,0.014555199444293976
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,8191,0.04678879976272583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,8191,0.02872479856014252
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,1,0.026767998933792114
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,1,0.02266400009393692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,3,0.02645600140094757
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,3,0.02268799990415573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,7,0.026815998554229736
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,8191,0.1621232032775879
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,7,0.022787199914455415
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,15,0.026704001426696777
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,31,0.02661440074443817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,31,0.022707200050354003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,63,0.026766398549079896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,63,0.022627200186252593
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,127,0.031374400854110716
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,127,0.02284640073776245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,255,0.04749279916286468
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,2047,0.012561599910259246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,255,0.02992959916591644
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,511,0.08339200019836426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,511,0.04776639938354492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,1023,0.08491680026054382
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,2047,0.2998208045959473
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,2047,0.1630560040473938
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,15,0.022891199588775633
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,4095,0.5882383823394776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,1,0.04328320026397705
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,4095,0.316156792640686
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,1,0.03710080087184906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,3,0.04340960085391998
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,7,0.04325760006904602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,3,0.03711679875850678
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,7,0.03707360029220581
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,15,0.037196800112724304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,31,0.044537600874900815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,31,0.03707840144634247
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,63,0.04535039961338043
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,127,0.0513759970664978
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,1023,0.15510239601135253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,127,0.03914560079574585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,255,0.0494704008102417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,511,0.15429760217666627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,511,0.08433600068092346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,1023,0.2989183902740479
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,1023,0.15917279720306396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,1,0.0780239999294281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,1,0.06655359864234925
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,3,0.07800480127334594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,3,0.06643040180206299
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,7,0.08003519773483277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,15,0.04335519969463349
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,7,0.06775040030479432
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,15,0.08067200183868409
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,15,0.06777920126914978
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,63,0.037187200784683225
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,31,0.08115519881248474
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,31,0.0677456021308899
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,63,0.08139039874076844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,63,0.06799839735031128
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,127,0.09412320256233216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,127,0.06936479806900024
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,255,0.08355519771575928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,255,0.09179199934005737
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,255,0.15613919496536255
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,1,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,1,0.00846719965338707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,3,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,511,0.2984911918640137
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,3,0.008350399881601333
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,511,0.15903520584106445
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,15,0.008588799834251403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,15,0.008343999832868576
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,7,0.008249600231647492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,31,0.008350399881601333
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,63,0.008529599756002426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,63,0.008417599648237229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,127,0.008473599702119828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,127,0.008380799740552902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,255,0.008486399799585343
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,255,0.008327999711036682
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,511,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,511,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,1023,0.012415999919176102
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,1023,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,2047,0.028723201155662535
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,2047,0.012455999851226807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,4095,0.04605759978294373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,4095,0.02731359899044037
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,8191,0.04588960111141205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,8191,0.08251039981842041
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,1,0.1497663974761963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,1,0.12522079944610595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,3,0.14974720478057862
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,3,0.12533919811248778
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,7,0.15178240537643434
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,15,0.15183199644088746
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,7,0.008321599662303924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,31,0.008515200018882752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,15,0.12730400562286376
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,31,0.12768640518188476
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,31,0.1520143985748291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,63,0.151910400390625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,63,0.12730560302734376
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,127,0.17216639518737792
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,255,0.3019488096237183
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,255,0.17268480062484742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,1,0.2923471927642822
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,1,0.2445375919342041
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,3,0.2947488069534302
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,3,0.24410879611968994
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,7,0.2944432020187378
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,7,0.24585440158843994
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,127,0.12962559461593628
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,15,0.29376959800720215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,15,0.24594559669494628
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,31,0.2942512035369873
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,31,0.24597439765930176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,7,0.12524800300598143
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,63,0.2951535940170288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,63,0.2460752010345459
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,1,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,1,0.008379200100898742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,127,0.34473600387573244
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,3,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,127,0.2501071929931641
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,7,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,15,0.009139200299978256
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,31,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,31,0.009139200299978256
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,63,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,63,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,127,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,127,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,255,0.010596799850463866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,255,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,511,0.012537600100040435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,511,0.010505600273609162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,1023,0.028896000981330872
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,1023,0.013107199966907502
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,2047,0.04615199863910675
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,2047,0.028676798939704894
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,4095,0.08224639892578126
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,4095,0.047353601455688475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,8191,0.15607039928436278
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,8191,0.08681600093841553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,3,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,7,0.008350399881601333
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,3,0.014534400403499603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,1,0.016495999693870545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,7,0.014643199741840363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,15,0.010507199913263321
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,15,0.015385599434375763
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,1,0.013724799454212188
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,31,0.014588800072669984
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,31,0.012750400602817536
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,63,0.01459839940071106
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,63,0.01337919980287552
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,7,0.014235199987888336
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,127,0.014182400703430176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,3,0.012539200484752655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,255,0.01858240067958832
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,255,0.012542399764060973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,511,0.039401599764823915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,511,0.018611200153827667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,1023,0.06748639941215515
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,1023,0.03913280069828033
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,2047,0.12434079647064208
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,4095,0.2392047882080078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,4095,0.12725759744644166
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,8191,0.47461438179016113
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,8191,0.2467871904373169
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,1,0.010623999685049058
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,127,0.014569599926471711
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,3,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,15,0.014449599385261535
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,7,0.010548800230026245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,2047,0.06823199987411499
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,15,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,3,0.008638399839401244
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,1,0.010476800054311753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,31,0.010476800054311753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,31,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,63,0.010467199981212616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,63,0.010356800258159637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,7,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,127,0.011622399836778641
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,127,0.010358399897813796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,255,0.010577599704265594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,255,0.008792000263929367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,511,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,511,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,1023,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,1023,0.010304000228643417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,2047,0.012455999851226807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,4095,0.012564800679683685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,15,0.00936639979481697
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,4095,0.012449599802494049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,8191,0.014481599628925323
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,8191,0.012563200294971466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,1,0.010486400127410889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,1,0.010543999820947647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,3,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,3,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,7,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,7,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,15,0.010502400249242783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,31,0.010521599650382995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,31,0.010367999970912933
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,63,0.010606399923563003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,63,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,127,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,127,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,255,0.010486400127410889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,255,0.010345599800348281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,511,0.010566399991512298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,511,0.010366400331258773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,1023,0.010601600259542465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,1023,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,2047,0.012608000636100769
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,2047,0.011383999884128571
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,4095,0.014521600306034088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,4095,0.012548799812793731
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,8191,0.018603199720382692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,2047,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,8191,0.014577600359916686
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,1,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,1,0.01056160032749176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,3,0.012495999783277511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,3,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,7,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,7,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,15,0.012399999797344208
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,15,0.010564800351858139
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,31,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,31,0.010585600137710571
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,63,0.012411200255155564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,63,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,127,0.01239359974861145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,127,0.010529600083827972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,255,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,511,0.018628799915313722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,511,0.012556800246238708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,1023,0.0390720009803772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,1023,0.01836480051279068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,2047,0.06596480011940002
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,2047,0.03856320083141327
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,4095,0.12115999460220336
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,4095,0.06779680252075196
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,8191,0.12556960582733154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,8191,0.23354880809783934
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,1,0.010580799728631973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,1,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,3,0.010569600015878677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,7,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,7,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,15,0.010590399801731109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,15,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,31,0.010500799864530563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,15,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,31,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,63,0.01058719977736473
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,63,0.010367999970912933
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,127,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,127,0.010332799702882766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,255,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,255,0.010548800230026245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,255,0.010320000350475311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,511,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,1023,0.012624000012874604
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,1023,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,2047,0.014281600713729858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,2047,0.010633599758148194
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,4095,0.01858240067958832
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,4095,0.012567999958992004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,8191,0.03990559875965118
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,8191,0.018665599822998046
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,1,0.022648000717163087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,1,0.018676799535751343
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,3,0.022575999796390533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,3,0.01867839992046356
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,7,0.018632000684738158
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,15,0.021265600621700288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,15,0.018614399433135986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,31,0.020732800662517547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,31,0.0186271995306015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,63,0.020662400126457214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,63,0.018592000007629395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,127,0.02415519952774048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,127,0.018628799915313722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,255,0.038996800780296326
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,3,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,255,0.02062080055475235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,511,0.06556320190429688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,511,0.03863359987735748
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,1023,0.11961760520935058
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,1023,0.06726239919662476
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,2047,0.22847359180450438
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,2047,0.1241312026977539
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,511,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,4095,0.24040799140930175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,4095,0.4508687973022461
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,7,0.022620800137519836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,1,0.03089759945869446
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,3,0.034980800747871396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,3,0.030934399366378783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,7,0.03505280017852783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,7,0.028958401083946227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,15,0.035067200660705566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,15,0.029598399996757507
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,31,0.035123199224472046
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,31,0.029382398724555968
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,63,0.03712320029735565
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,63,0.029422399401664735
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,127,0.043156799674034116
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,127,0.03091199994087219
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,255,0.0663424015045166
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,511,0.11955679655075073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,511,0.06684960126876831
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,1023,0.22765920162200928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,1023,0.12443519830703735
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,1,0.06177279949188232
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,1,0.05251200199127197
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,3,0.061638402938842776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,3,0.05154719948768616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,1,0.03508319854736328
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,7,0.06161919832229614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,7,0.052560001611709595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,15,0.06160479784011841
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,15,0.051553601026535036
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,31,0.06376960277557372
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,31,0.05345439910888672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,63,0.06374559998512268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,63,0.05356320142745972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,127,0.07273439764976501
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,127,0.05543839931488037
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,255,0.03980480134487152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,511,0.22766718864440919
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,511,0.12244640588760376
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,1,0.00835840031504631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,1,0.008419200032949447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,3,0.008372800052165985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,3,0.0083856001496315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,7,0.008408000320196151
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,7,0.008353599905967712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,15,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,15,0.008371199667453765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,31,0.008551999926567078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,31,0.008414400368928909
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,63,0.008363199979066848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,63,0.008371199667453765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,127,0.008374399691820144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,127,0.00835679993033409
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,255,0.008350399881601333
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,511,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,511,0.008404800295829773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,1023,0.012383999675512314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,1023,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,2047,0.01902559995651245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,2047,0.012566399574279786
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,4095,0.03785600066184998
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,4095,0.018702399730682374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,8191,0.06964319944381714
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,8191,0.03714720010757446
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,1,0.11476000547409057
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,1,0.09648000001907349
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,255,0.11960320472717285
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,3,0.0966048002243042
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,7,0.11496479511260986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,7,0.09662399888038635
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,15,0.11704800128936768
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,15,0.09653279781341553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,255,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,31,0.11694719791412353
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,31,0.09877600073814392
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,63,0.11697280406951904
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,63,0.09872959852218628
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,127,0.13226720094680786
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,127,0.1023743987083435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,255,0.22957119941711426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,255,0.13020000457763672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,1,0.22128961086273194
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,255,0.07013919949531555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,1,0.1865391969680786
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,3,0.18663040399551392
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,7,0.18671200275421143
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,15,0.223307204246521
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,15,0.18846559524536133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,31,0.22317759990692138
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,31,0.18717119693756104
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,63,0.22369120121002198
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,63,0.1887439966201782
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,3,0.11480159759521484
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,3,0.2224047899246216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,1,0.010503999888896942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,127,0.26035199165344236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,127,0.19378559589385985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,1,0.010086400061845779
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,3,0.010305599868297577
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,3,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,7,0.22333600521087646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,7,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,7,0.008531200140714646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,15,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,15,0.01000640019774437
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,31,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,31,0.00846719965338707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,63,0.010311999917030334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,63,0.00989919975399971
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,127,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,127,0.008582399785518646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,255,0.010476800054311753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,255,0.008472000062465668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,511,0.012507200241088867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,511,0.01048320010304451
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,1023,0.01891999989748001
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,1023,0.012563200294971466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,2047,0.04129279851913452
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,2047,0.018638400733470915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,4095,0.07189120054244995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,4095,0.041526401042938234
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,8191,0.13256959915161132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,1,0.012608000636100769
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,8191,0.07430400252342224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,3,0.012638400495052337
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,7,0.012569600343704223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,1,0.01053759977221489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,15,0.012548799812793731
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,7,0.011124800145626067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,15,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,31,0.012503999471664428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,31,0.010553599894046783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,63,0.012559999525547028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,63,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,127,0.012561599910259246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,127,0.010523200035095215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,255,0.012691199779510498
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,255,0.01168000027537346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,511,0.028964799642562867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,511,0.014569599926471711
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,1023,0.029083201289176942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,2047,0.0836463987827301
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,2047,0.048065599799156186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,4095,0.15833439826965331
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,4095,0.08746399879455566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,8191,0.306712007522583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,8191,0.1635696053504944
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,3,0.010694400221109391
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,16383,0.6046288013458252
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,16383,0.31950719356536866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,1,0.009015999734401703
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,3,0.00921119973063469
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,1023,0.04742879867553711
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,7,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,15,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,3,0.008473599702119828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,1,0.010344000160694122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,31,0.01032159999012947
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,31,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,63,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,7,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,63,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,127,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,127,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,255,0.010366400331258773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,255,0.0094991996884346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,511,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,511,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,1023,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,2047,0.010526400059461594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,2047,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,4095,0.012587200105190276
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,4095,0.012513600289821625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,8191,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,16383,0.016569599509239197
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,16383,0.014686399698257446
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,1,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,1,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,3,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,3,0.009006399661302567
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,7,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,7,0.008724799752235413
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,15,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,15,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,31,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,31,0.010494399815797806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,63,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,63,0.009523200243711472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,127,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,127,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,255,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,15,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,255,0.010344000160694122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,511,0.010486400127410889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,511,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,1023,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,1023,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,1023,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,2047,0.010550399869680404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,2047,0.0104592002928257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,4095,0.012513600289821625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,8191,0.014534400403499603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,8191,0.014531199634075165
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,8191,0.012705600261688233
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,16383,0.0160863995552063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,1,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,1,0.008723200112581254
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,3,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,3,0.010329599678516387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,7,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,7,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,15,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,15,0.008476799726486206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,31,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,31,0.00931359976530075
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,63,0.010478399693965912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,63,0.009409599751234055
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,127,0.01053439974784851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,127,0.009935999661684037
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,255,0.01056319996714592
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,255,0.010345599800348281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,511,0.012646399438381195
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,511,0.0106175996363163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,1023,0.028916800022125246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,1023,0.014398400485515595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,2047,0.0471343994140625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,4095,0.012705600261688233
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,2047,0.02901119887828827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,4095,0.08225600123405456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,4095,0.047030401229858396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,16383,0.030959999561309813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,8191,0.08590880036354065
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,8191,0.15457439422607422
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,1,0.01011200025677681
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,1,0.008540800213813782
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,16383,0.29918880462646485
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,3,0.009841600060462951
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,7,0.010279999673366546
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,7,0.008609600365161896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,15,0.009732799977064133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,15,0.008531200140714646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,31,0.01014079973101616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,31,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,63,0.008899199962615966
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,63,0.008555199950933456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,127,0.008947200328111648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,127,0.008513599634170532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,255,0.008499199897050858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,511,0.008574400097131729
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,511,0.008489599823951722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,1023,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,1023,0.010875199735164643
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,2047,0.010505600273609162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,2047,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,4095,0.01255040019750595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,4095,0.010729599744081497
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,8191,0.02881760001182556
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,8191,0.01438080072402954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,16383,0.04664640128612518
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,16383,0.1652176022529602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,16383,0.02908799946308136
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,1,0.01462559998035431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,1,0.016771200299263
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,3,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,3,0.016899199783802034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,3,0.014591999351978302
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,7,0.014591999351978302
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,15,0.016680000722408293
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,15,0.014643199741840363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,31,0.01672479957342148
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,31,0.014667199552059173
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,63,0.016734400391578676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,63,0.014555199444293976
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,127,0.016697600483894348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,127,0.014596800506114959
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,255,0.02892960011959076
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,255,0.014561599493026734
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,511,0.04652160108089447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,511,0.02890239953994751
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,1023,0.08254560232162475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,1023,0.04762240052223206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,2047,0.15460000038146973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,2047,0.08536800146102905
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,255,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,4095,0.16296160221099854
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,4095,0.30034561157226564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,1,0.025972801446914672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,1,0.02269279956817627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,8191,0.590993595123291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,3,0.026715201139450074
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,8191,0.3193392038345337
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,7,0.02670240104198456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,3,0.022673599421977997
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,15,0.026731199026107787
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,15,0.022726400196552275
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,7,0.022756800055503845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,31,0.026833599805831908
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,31,0.02271360009908676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,63,0.026358398795127868
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,63,0.022699199616909027
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,127,0.031079998612403868
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,127,0.022708800435066224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,255,0.04603039920330047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,255,0.030351999402046203
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,511,0.08233759999275207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,7,0.016550399363040924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,511,0.04768800139427185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,1023,0.08485599756240844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,2047,0.3000463962554932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,2047,0.15945600271224974
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,1,0.043268799781799316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,1,0.037001600861549376
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,3,0.03709760010242462
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,7,0.04321280121803284
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,7,0.03717440068721771
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,15,0.04321599900722504
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,15,0.03707520067691803
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,31,0.044019201397895814
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,31,0.03707840144634247
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,63,0.04527199864387512
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,63,0.03710080087184906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,127,0.05159040093421936
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,127,0.03983680009841919
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,255,0.08223360180854797
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,255,0.050748801231384276
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,511,0.15434399843215943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,1023,0.15405280590057374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,511,0.08489120006561279
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,1023,0.2982016086578369
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,1023,0.16023999452590942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,1,0.010340800136327743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,1,0.008635199815034866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,3,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,3,0.008393599838018417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,7,0.008408000320196151
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,15,0.008369600027799606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,15,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,31,0.008427199721336365
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,31,0.008419200032949447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,63,0.00843840017914772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,63,0.008398400247097015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,127,0.008478400111198426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,127,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,255,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,255,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,511,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,511,0.010299199819564819
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,1023,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,1023,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,2047,0.012529599666595458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,2047,0.01056160032749176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,4095,0.028753599524497984
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,4095,0.014446400105953217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,8191,0.046158400177955625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,8191,0.028705599904060363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,16383,0.08241919875144958
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,16383,0.0468207985162735
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,1,0.08006399869918823
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,1,0.06781439781188965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,3,0.07923679947853088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,3,0.06782559752464294
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,7,0.07811520099639893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,7,0.06785759925842286
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,15,0.08002079725265503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,3,0.043249601125717164
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,31,0.08014400005340576
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,31,0.06779680252075196
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,63,0.08035359978675842
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,7,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,63,0.06790879964828492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,127,0.09416000247001648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,127,0.07200160026550292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,255,0.1553279995918274
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,255,0.09313759803771973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,1,0.1496991991996765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,1,0.12521760463714598
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,3,0.1497648000717163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,3,0.12519359588623047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,7,0.15169119834899902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,7,0.12514079809188844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,15,0.1518623948097229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,15,0.12719039916992186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,31,0.15175199508666992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,31,0.12724000215530396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,15,0.06771360039710998
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,63,0.12730720043182372
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,127,0.17715840339660643
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,127,0.13192800283432007
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,511,0.1588912010192871
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,255,0.1778480052947998
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,255,0.30253279209136963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,1,0.00841279998421669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,1,0.008449599891901017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,3,0.008372800052165985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,7,0.008484800159931184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,7,0.00828320011496544
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,15,0.0084927998483181
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,63,0.15324800014495848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,15,0.008366400003433227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,31,0.008406399935483932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,31,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,63,0.008372800052165985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,63,0.008393599838018417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,127,0.008379200100898742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,511,0.29876959323883057
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,255,0.008502399921417237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,255,0.008316799998283386
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,511,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,3,0.008371199667453765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,511,0.00852160006761551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,1023,0.0104592002928257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,2047,0.027377599477767946
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,2047,0.01255359947681427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,4095,0.045428800582885745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,4095,0.028095999360084535
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,8191,0.08148800134658814
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,8191,0.046147200465202334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,16383,0.15281920433044432
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,16383,0.08309760093688964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,1,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,1,0.00867680013179779
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,3,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,7,0.010318399965763092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,127,0.008393599838018417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,7,0.009576000273227692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,15,0.010480000078678131
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,15,0.010078399628400802
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,3,0.01048159971833229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,31,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,31,0.010096000134944915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,63,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,127,0.010342399775981902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,127,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,255,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,255,0.009699200093746186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,511,0.012617599964141846
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,1023,0.010516799986362457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,511,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,2047,0.046219199895858765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,1023,0.012540799379348756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,2047,0.02809920012950897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,4095,0.047092801332473753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,8191,0.1553279995918274
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,8191,0.08317440152168273
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,16383,0.30046560764312746
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,16383,0.15878720283508302
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,32767,0.5900976181030273
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,63,0.008390399813652038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,1,0.009232000261545182
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,1023,0.028439998626708984
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,3,0.010497599840164185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,32767,0.30856320858001707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,7,0.010375999659299851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,4095,0.08211519718170165
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,15,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,3,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,1,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,31,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,31,0.008425600081682205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,63,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,7,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,127,0.01032159999012947
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,127,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,15,0.009254399687051773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,255,0.009430400282144546
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,511,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,511,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,1023,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,1023,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,2047,0.01051200032234192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,2047,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,4095,0.011297599971294403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,4095,0.01069599986076355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,8191,0.01459999978542328
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,8191,0.013555200397968292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,16383,0.016542400419712066
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,16383,0.016036799550056456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,32767,0.018612800538539885
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,32767,0.016667200624942778
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,1,0.009468799829483033
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,1,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,3,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,7,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,7,0.010344000160694122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,15,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,15,0.010311999917030334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,31,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,31,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,63,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,63,0.008953599631786347
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,127,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,127,0.010351999849081039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,255,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,255,0.010344000160694122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,511,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,63,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,511,0.01027199998497963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,255,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,1023,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,2047,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,2047,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,4095,0.012455999851226807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,4095,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,8191,0.014563199877738953
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,8191,0.012449599802494049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,3,0.009379199892282485
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,16383,0.01653600037097931
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,16383,0.014632000029087067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,32767,0.03212159872055054
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,32767,0.01659359931945801
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,1,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,1,0.008398400247097015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,3,0.008470399677753449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,3,0.008427199721336365
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,7,0.008348800241947174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,7,0.008379200100898742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,15,0.008348800241947174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,15,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,31,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,31,0.008470399677753449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,63,0.008380799740552902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,63,0.008404800295829773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,127,0.008390399813652038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,127,0.008350399881601333
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,255,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,255,0.00846880003809929
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,511,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,511,0.0084927998483181
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,1023,0.01244639977812767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,1023,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,2047,0.027480000257492067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,2047,0.012537600100040435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,4095,0.04555839896202087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,4095,0.02757279872894287
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,8191,0.08099039793014526
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,1023,0.010351999849081039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,16383,0.15293600559234619
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,16383,0.08220800161361694
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,32767,0.2955647945404053
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,32767,0.15385279655456544
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,1,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,1,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,3,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,3,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,7,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,7,0.010276799649000167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,15,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,15,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,31,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,31,0.010356800258159637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,63,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,63,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,127,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,127,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,255,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,255,0.010291200131177902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,511,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,511,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,1023,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,1023,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,2047,0.010480000078678131
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,2047,0.010633599758148194
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,4095,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,4095,0.012379200011491776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,8191,0.014550399780273438
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,8191,0.01295360028743744
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,16383,0.01658560037612915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,16383,0.030955201387405394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,32767,0.03089280128479004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,1,0.012451200187206269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,1,0.012099199742078782
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,3,0.012547199428081513
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,3,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,7,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,8191,0.04603999853134155
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,7,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,15,0.012435200065374375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,31,0.012478400021791458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,31,0.011420799791812897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,63,0.012481600046157837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,63,0.012385600060224534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,127,0.012449599802494049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,127,0.012387199699878693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,255,0.012427199631929398
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,255,0.012406399846076966
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,511,0.02885279953479767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,511,0.014655999839305878
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,1023,0.04737440049648285
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,1023,0.028990399837493897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,2047,0.08384000062942505
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,2047,0.04790880084037781
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,4095,0.15789439678192138
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,4095,0.08470720052719116
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,8191,0.15858399868011475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,8191,0.3067248106002808
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,16383,0.30551040172576904
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,16383,0.6054224014282227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,32767,0.048368000984191896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,1,0.01462559998035431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,3,0.01669439971446991
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,3,0.01465120017528534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,7,0.016711999475955964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,15,0.012511999905109405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,7,0.014654399454593658
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,15,0.016543999314308167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,15,0.014646400511264802
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,31,0.016575999557971954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,31,0.014672000706195832
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,63,0.01653439998626709
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,63,0.014660799503326416
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,127,0.016641600430011748
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,127,0.014587199687957764
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,255,0.028112000226974486
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,255,0.014561599493026734
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,511,0.045747199654579164
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,1023,0.0819920003414154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,1023,0.046768000721931456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,2047,0.15471839904785156
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,2047,0.08320639729499817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,4095,0.30083839893341063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,4095,0.15635520219802856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,1,0.01653279960155487
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,1,0.02483679950237274
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,1,0.022734400629997254
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,3,0.026825600862503053
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,3,0.022697600722312927
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,7,0.026419198513031004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,7,0.022785599529743194
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,15,0.02601119875907898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,15,0.022761599719524385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,31,0.02648800015449524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,31,0.022710399329662324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,63,0.024881599843502043
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,63,0.02271520048379898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,127,0.02996639907360077
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,127,0.022707200050354003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,255,0.046260800957679746
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,255,0.02938719987869263
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,511,0.08171359896659851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,511,0.04749279916286468
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,1023,0.1546880006790161
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,1023,0.08363839983940125
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,2047,0.15576800107955932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,2047,0.2991744041442871
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,1,0.008422400057315826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,1,0.008425600081682205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,3,0.008366400003433227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,7,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,3,0.008779200166463852
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,7,0.008433599770069123
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,15,0.00841120034456253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,15,0.008550400286912918
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,31,0.008459199965000153
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,31,0.008455999940633774
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,63,0.008489599823951722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,63,0.008406399935483932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,127,0.008452799916267396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,255,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,255,0.0083856001496315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,511,0.008423999696969987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,511,0.008369600027799606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,1023,0.009531199932098389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,1023,0.00846719965338707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,2047,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,2047,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,4095,0.012521600723266602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,4095,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,8191,0.028446400165557863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,8191,0.01446560025215149
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,16383,0.028963199257850646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,32767,0.08173760175704955
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,32767,0.04656479954719543
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,1,0.04323039948940277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,1,0.037062400579452516
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,3,0.043321600556373595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,3,0.03710559904575348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,7,0.0433135986328125
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,7,0.03712800145149231
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,15,0.043208000063896176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,15,0.03707360029220581
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,31,0.04410400092601776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,31,0.03712640106678009
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,63,0.04536640048027039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,63,0.03704319894313812
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,127,0.05104799866676331
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,127,0.04007999897003174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,127,0.008454400300979614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,255,0.050177597999572755
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,511,0.15469599962234498
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,511,0.08412160277366638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,1023,0.29923999309539795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,1023,0.1567088007926941
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,511,0.029054400324821473
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,1,0.08010079860687255
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,1,0.06786720156669616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,3,0.07997919917106629
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,3,0.06778079867362977
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,7,0.0798479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,7,0.0678384006023407
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,15,0.08019520044326782
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,15,0.06784319877624512
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,31,0.08042399883270264
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,31,0.06783679723739625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,63,0.08196480274200439
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,255,0.08303040266036987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,63,0.06993600130081176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,127,0.07202240228652954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,127,0.09430559873580932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,255,0.15643199682235717
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,16383,0.046014401316642764
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,511,0.15740959644317626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,511,0.2991951942443848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,1,0.008423999696969987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,1,0.009430400282144546
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,3,0.008491200208663941
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,7,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,7,0.008555199950933456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,15,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,15,0.008561599999666214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,31,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,31,0.008790399879217148
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,63,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,63,0.01032480001449585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,127,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,127,0.01029599979519844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,255,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,255,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,511,0.010865599662065507
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,511,0.010299199819564819
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,1023,0.01067200005054474
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,1023,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,2047,0.012614400684833526
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,255,0.09321280121803284
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,2047,0.010590399801731109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,4095,0.028883200883865357
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,8191,0.04649440050125122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,8191,0.028886398673057555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,16383,0.08162239789962769
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,3,0.008416000008583068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,16383,0.046876800060272214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,32767,0.08254719972610473
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,32767,0.15442559719085694
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,3,0.00840959995985031
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,1,0.00843840017914772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,7,0.0083856001496315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,3,0.008395200222730636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,7,0.008423999696969987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,15,0.008404800295829773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,15,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,31,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,63,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,63,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,127,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,127,0.008396799862384795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,255,0.008444800227880477
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,255,0.008417599648237229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,511,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,511,0.00846719965338707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,1023,0.010503999888896942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,1023,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,2047,0.027447998523712158
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,2047,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,4095,0.014480000734329224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,4095,0.02865920066833496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,8191,0.08229119777679443
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,8191,0.04615040123462677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,1,0.008585599809885025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,16383,0.15282560586929322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,16383,0.0828831970691681
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,32767,0.29556000232696533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,31,0.00836160033941269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,32767,0.15482399463653565
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,1,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,65535,0.3018687963485718
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,65535,0.5810815811157226
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,3,0.008417599648237229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,7,0.008483199775218964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,4095,0.04580000042915344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,15,0.010500799864530563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,1,0.01032159999012947
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,31,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,31,0.008479999750852585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,63,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,63,0.008535999804735184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,127,0.010329599678516387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,127,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,255,0.01032159999012947
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,255,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,511,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,511,0.010313600301742554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,1023,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,1023,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,2047,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,2047,0.010478399693965912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,4095,0.010518400371074677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,4095,0.010633599758148194
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,8191,0.014443199336528777
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,3,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,8191,0.012604799866676331
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,16383,0.017662400007247926
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,16383,0.016715200245380403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,15,0.00926240012049675
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,32767,0.018596799671649934
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,65535,0.020580799877643587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,65535,0.01873279958963394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,1,0.009464000165462495
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,1,0.008475200086832047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,3,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,3,0.008479999750852585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,7,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,7,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,15,0.009239999949932099
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,15,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,31,0.008479999750852585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,31,0.008446399867534638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,63,0.008417599648237229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,63,0.00841279998421669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,127,0.008393599838018417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,127,0.008452799916267396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,255,0.008504000306129456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,7,0.008511999994516373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,255,0.008390399813652038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,511,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,511,0.00843840017914772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,1023,0.01048320010304451
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,1023,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,2047,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,2047,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,4095,0.011726400256156922
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,32767,0.01865600049495697
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,4095,0.010550399869680404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,8191,0.014499199390411378
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,8191,0.01419840008020401
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,16383,0.016655999422073364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,16383,0.014644800126552582
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,32767,0.018764799833297728
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,32767,0.016551999747753142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,65535,0.03493280112743378
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,65535,0.018564799427986146
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,1,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,1,0.008502399921417237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,3,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,3,0.008484800159931184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,7,0.008654399961233138
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,7,0.01029599979519844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,15,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,15,0.008408000320196151
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,31,0.009195200353860854
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,31,0.010328000038862228
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,63,0.008449599891901017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,127,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,127,0.009412799775600434
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,255,0.009003199636936188
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,511,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,511,0.009161599725484849
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,1023,0.010598400235176086
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,1023,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,2047,0.012555199861526489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,2047,0.012134400010108948
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,4095,0.028772801160812378
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,4095,0.014473600685596466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,8191,0.029014399647712706
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,8191,0.047056001424789426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,16383,0.08231199979782104
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,32767,0.15360959768295288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,32767,0.08315680027008057
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,65535,0.29644320011138914
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,65535,0.15488159656524658
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,1,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,1,0.010283199697732925
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,3,0.010332799702882766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,3,0.010308799892663955
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,7,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,63,0.008483199775218964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,7,0.01032159999012947
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,15,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,15,0.010337600111961364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,31,0.010363200306892395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,63,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,63,0.010299199819564819
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,127,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,127,0.01034879982471466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,255,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,255,0.010315199941396713
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,16383,0.04726240038871765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,511,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,1023,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,1023,0.010307200253009796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,2047,0.010633599758148194
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,2047,0.010486400127410889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,4095,0.012465599924325943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,4095,0.012345600128173827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,8191,0.014497600495815277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,8191,0.012601600587368011
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,16383,0.01652960032224655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,16383,0.014569599926471711
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,32767,0.031839999556541446
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,255,0.008780799806118011
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,32767,0.016523200273513793
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,65535,0.04983839988708496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,65535,0.032995200157165526
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,1,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,1,0.008473599702119828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,3,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,511,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,3,0.008449599891901017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,7,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,15,0.010305599868297577
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,15,0.01029599979519844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,31,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,31,0.010320000350475311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,63,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,63,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,127,0.01061279997229576
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,127,0.009668800234794616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,255,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,255,0.009440000355243682
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,511,0.01257600039243698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,511,0.010545600205659866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,1023,0.027851200103759764
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,1023,0.012703999876976013
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,2047,0.04617919921875
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,2047,0.028859201073646545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,31,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,4095,0.046865600347518924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,8191,0.15433119535446166
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,8191,0.08386399745941162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,7,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,16383,0.297871994972229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,16383,0.1580672025680542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,1,0.012436799705028534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,1,0.010529600083827972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,3,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,3,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,32767,0.3057823896408081
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,7,0.012521600723266602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,32767,0.5854400157928467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,15,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,7,0.012467200309038163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,31,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,31,0.010539200156927109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,15,0.012435200065374375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,63,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,127,0.012537600100040435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,127,0.010520000010728836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,255,0.012656000256538392
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,255,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,511,0.02794559895992279
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,511,0.014521600306034088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,1023,0.046291199326515195
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,1023,0.029068800806999206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,2047,0.08380159735679626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,4095,0.08269760012626648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,2047,0.04732320010662079
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,4095,0.1566256046295166
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,4095,0.08461120128631591
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,1,0.01661760061979294
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,8191,0.3021967887878418
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,1,0.014591999351978302
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,8191,0.15865600109100342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,3,0.014603200554847717
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,3,0.016604800522327424
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,7,0.016686399281024934
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,7,0.01465280055999756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,15,0.016572800278663636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,15,0.015868799388408662
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,31,0.01451520025730133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,63,0.016675199568271636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,63,0.014616000652313232
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,127,0.016697600483894348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,127,0.01467359960079193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,255,0.027628800272941588
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,255,0.014529600739479065
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,511,0.02887519896030426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,1023,0.0817520022392273
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,1023,0.04757120013237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,2047,0.15268319845199585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,63,0.012540799379348756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,2047,0.0837664008140564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,4095,0.29488000869750974
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,4095,0.15730559825897217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,1,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,1,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,3,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,3,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,7,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,15,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,15,0.008371199667453765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,31,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,31,0.009622400254011154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,63,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,63,0.009920000284910201
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,127,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,31,0.016631999611854555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,127,0.010302399843931198
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,255,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,255,0.009673599898815156
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,511,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,511,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,1023,0.010542400181293488
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,511,0.045926401019096376
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,2047,0.011448000371456147
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,2047,0.010662399977445603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,4095,0.012544000148773193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,4095,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,8191,0.014659200608730317
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,8191,0.013092799484729767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,16383,0.030883198976516722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,16383,0.016545599699020384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,32767,0.04879679977893829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,32767,0.030895999073982237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,7,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,65535,0.08414559960365295
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,65535,0.04932160079479218
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,1,0.02624639868736267
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,1,0.022711999714374542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,3,0.02277279943227768
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,7,0.024823999404907225
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,7,0.022711999714374542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,15,0.024771200120449068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,15,0.02268960028886795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,31,0.0264847993850708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,31,0.02272319942712784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,63,0.02653439939022064
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,63,0.022737599909305573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,1023,0.010667199641466141
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,127,0.030167999863624572
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,127,0.022776000201702118
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,255,0.04588960111141205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,255,0.031038400530815125
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,511,0.08141440153121948
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,511,0.04789440035820007
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,1023,0.15235519409179688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,1023,0.08416479825973511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,2047,0.1564703941345215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,2047,0.294654393196106
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,1,0.043252798914909366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,1,0.03910239934921265
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,3,0.04334079921245575
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,3,0.039052799344062805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,7,0.04326080083847046
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,7,0.03919360041618347
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,15,0.04326240122318268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,15,0.0391184002161026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,31,0.043268799781799316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,3,0.026763200759887695
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,63,0.04532319903373718
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,63,0.039108800888061526
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,127,0.05135679841041565
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,127,0.04150719940662384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,255,0.08263199925422668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,255,0.051692801713943484
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,511,0.1530192017555237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,511,0.08561279773712158
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,1,0.009492799639701843
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,1,0.00849440023303032
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,1023,0.15763200521469117
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,1023,0.2949919939041138
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,3,0.009286399930715561
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,3,0.008488000184297562
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,7,0.008550400286912918
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,15,0.008479999750852585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,31,0.008556800335645676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,31,0.008548799902200699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,63,0.008710400015115739
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,63,0.008548799902200699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,127,0.008769600093364716
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,127,0.008502399921417237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,255,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,255,0.0084927998483181
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,511,0.010211200267076493
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,511,0.00849440023303032
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,1023,0.01029599979519844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,1023,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,2047,0.01051200032234192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,2047,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,4095,0.012624000012874604
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,4095,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,8191,0.028409600257873535
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,8191,0.01448799967765808
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,16383,0.04610080122947693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,16383,0.02885119915008545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,32767,0.08178719878196716
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,32767,0.04720639884471893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,7,0.008563199639320373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,15,0.008660800009965896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,65535,0.08326719999313355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,1,0.008574400097131729
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,3,0.00838399976491928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,31,0.03912799954414368
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,7,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,65535,0.15348000526428224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,15,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,1,0.008550400286912918
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,31,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,31,0.008609600365161896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,63,0.010332799702882766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,15,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,63,0.009201599657535553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,127,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,127,0.009319999814033508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,255,0.010363200306892395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,255,0.010351999849081039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,511,0.01034879982471466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,511,0.010467199981212616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,1023,0.01058880016207695
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,1023,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,2047,0.012532800436019897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,2047,0.012452799826860428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,7,0.008550400286912918
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,4095,0.028971201181411742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,4095,0.013758400082588195
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,8191,0.046435201168060304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,8191,0.02868480086326599
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,16383,0.04657599925994873
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,32767,0.15315359830856323
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,32767,0.08245599865913392
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,65535,0.296345591545105
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,3,0.008444800227880477
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,65535,0.15372320413589477
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,131071,0.5825247764587402
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,131071,0.29714241027832033
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,1,0.008951999992132188
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,3,0.008606400340795517
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,16383,0.08291360139846801
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,7,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,15,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,1,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,3,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,31,0.008430399745702744
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,31,0.00846880003809929
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,63,0.00841279998421669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,63,0.008404800295829773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,127,0.008396799862384795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,127,0.00841120034456253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,255,0.008500800281763077
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,255,0.008393599838018417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,511,0.008478400111198426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,511,0.00841279998421669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,1023,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,1023,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,15,0.008564800024032593
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,2047,0.009649600088596343
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,4095,0.01056160032749176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,4095,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,8191,0.012513600289821625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,8191,0.012529599666595458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,16383,0.01653759926557541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,16383,0.014564800262451171
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,32767,0.018136000633239745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,32767,0.016524800658226015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,65535,0.016979199647903443
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,131071,0.02274080067873001
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,131071,0.02069759964942932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,1,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,1,0.008483199775218964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,3,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,3,0.0084927998483181
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,7,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,7,0.00851999968290329
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,15,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,15,0.00843840017914772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,31,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,31,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,63,0.009483200311660767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,63,0.008366400003433227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,127,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,127,0.008942399919033051
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,255,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,255,0.008380799740552902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,2047,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,511,0.008977600187063218
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,1023,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,1023,0.008483199775218964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,2047,0.010502400249242783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,2047,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,65535,0.018739199638366698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,4095,0.01053600013256073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,4095,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,8191,0.012577599287033081
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,8191,0.012751999497413635
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,16383,0.017638400197029114
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,16383,0.016710400581359863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,32767,0.018587200343608855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,32767,0.018587200343608855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,65535,0.020428800582885744
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,65535,0.01871519982814789
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,131071,0.039526399970054624
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,131071,0.022566400468349457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,1,0.008408000320196151
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,1,0.008499199897050858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,3,0.008515200018882752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,7,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,7,0.008532799780368805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,15,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,15,0.008563199639320373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,31,0.010307200253009796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,7,0.008878400176763534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,31,0.008491200208663941
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,63,0.010311999917030334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,63,0.008459199965000153
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,127,0.008646400272846222
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,127,0.008510400354862214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,255,0.008551999926567078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,255,0.008504000306129456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,511,0.00849440023303032
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,511,0.00852160006761551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,1023,0.010265599936246872
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,3,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,1023,0.008684799820184708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,2047,0.010558400303125381
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,4095,0.012569600343704223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,4095,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,8191,0.02884320020675659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,8191,0.01456640064716339
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,16383,0.04599039852619171
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,16383,0.02889760136604309
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,32767,0.08231520056724548
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,32767,0.046081599593162534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,65535,0.1542240023612976
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,511,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,65535,0.08262400031089782
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,131071,0.29599359035491946
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,131071,0.15490399599075316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,1,0.00846719965338707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,1,0.008540800213813782
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,3,0.008428800106048583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,3,0.008416000008583068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,7,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,7,0.009515199810266495
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,15,0.008371199667453765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,15,0.009513600170612336
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,31,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,31,0.009643200039863586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,63,0.00921119973063469
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,63,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,127,0.009332799911499023
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,127,0.00846719965338707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,255,0.0083856001496315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,511,0.009169600158929824
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,2047,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,511,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,1023,0.009566400200128555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,2047,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,1023,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,2047,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,4095,0.010864000022411346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,8191,0.014560000598430633
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,8191,0.01401599943637848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,16383,0.014526399970054626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,32767,0.018643200397491455
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,32767,0.016625599563121797
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,65535,0.03510720133781433
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,65535,0.018598400056362152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,131071,0.053144001960754396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,131071,0.03553279936313629
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,1,0.0083856001496315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,1,0.008348800241947174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,3,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,3,0.008364800363779068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,7,0.00840959995985031
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,7,0.008345600217580795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,15,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,15,0.008270400017499924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,31,0.00835679993033409
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,31,0.008342400193214417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,63,0.008364800363779068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,63,0.008367999643087386
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,255,0.010651200264692306
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,127,0.008343999832868576
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,255,0.008511999994516373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,4095,0.010678400099277497
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,255,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,16383,0.01656640022993088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,511,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,1023,0.01250240057706833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,1023,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,2047,0.02770400047302246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,2047,0.012628799676895142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,4095,0.04553439915180206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,4095,0.028148800134658813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,8191,0.08180959820747376
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,8191,0.04572640061378479
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,16383,0.1525488018989563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,16383,0.08198400139808655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,32767,0.15363839864730836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,32767,0.2951632022857666
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,65535,0.295964789390564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,65535,0.5801407814025878
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,1,0.010334400087594986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,127,0.008398400247097015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,3,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,3,0.010225600004196167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,7,0.010507199913263321
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,7,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,15,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,15,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,31,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,31,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,511,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,63,0.010363200306892395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,63,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,127,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,127,0.010308799892663955
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,255,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,255,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,511,0.012503999471664428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,511,0.010732799768447876
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,1023,0.027977600693702698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,1023,0.013065600395202636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,2047,0.04589279890060425
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,2047,0.028700798749923706
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,4095,0.08206719756126404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,4095,0.04615199863910675
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,8191,0.15466079711914063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,8191,0.08314719796180725
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,16383,0.15515040159225463
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,16383,0.29853599071502684
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,1,0.012532800436019897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,3,0.012455999851226807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,1,0.012404800206422806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,1,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,3,0.012414400279521943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,7,0.012534399330615998
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,7,0.011310400068759918
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,15,0.012673600018024445
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,31,0.012505599856376648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,31,0.012323199957609176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,63,0.012547199428081513
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,63,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,127,0.012348800152540206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,255,0.012598399817943574
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,255,0.012403199821710587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,511,0.027988800406455995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,511,0.014572800695896148
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,1023,0.04724799990653992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,1023,0.02911199927330017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,2047,0.08370239734649658
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,2047,0.047279998660087585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,4095,0.1570032000541687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,4095,0.08390719890594482
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,8191,0.3030816078186035
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,15,0.012380799651145935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,8191,0.15665440559387206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,1,0.009848000109195709
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,3,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,3,0.010308799892663955
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,7,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,7,0.010255999863147736
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,15,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,31,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,31,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,63,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,63,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,127,0.010289599746465683
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,127,0.01035040020942688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,255,0.010329599678516387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,255,0.010342399775981902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,511,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,511,0.010267200320959092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,1023,0.010326399654150008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,1023,0.010286399722099304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,2047,0.01066880002617836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,2047,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,4095,0.012336000055074691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,4095,0.012401600182056428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,8191,0.01451520025730133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,8191,0.01257600039243698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,16383,0.01652639955282211
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,16383,0.01456640064716339
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,32767,0.03205280005931854
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,32767,0.016513599455356597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,1,0.010569600015878677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,65535,0.050335997343063356
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,65535,0.033051198720932005
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,131071,0.08559839725494385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,15,0.010316800326108932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,1,0.01664479970932007
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,1,0.01650719940662384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,3,0.01669439971446991
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,3,0.016633599996566772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,7,0.016708800196647645
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,7,0.016484799981117248
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,15,0.01669919937849045
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,15,0.016620799899101257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,31,0.01668799966573715
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,31,0.016451199352741242
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,63,0.01664479970932007
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,63,0.016590400040149687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,127,0.016740800440311433
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,127,0.016545599699020384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,255,0.02828960120677948
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,255,0.016484799981117248
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,511,0.04599199891090393
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,511,0.029206401109695433
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,1023,0.08147199749946595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,1023,0.04742879867553711
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,2047,0.15270400047302246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,2047,0.08322880268096924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,4095,0.29489281177520754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,4095,0.1550495982170105
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,1,0.026785600185394286
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,1,0.024779200553894043
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,3,0.02680639922618866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,3,0.024700799584388734
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,7,0.02675040066242218
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,7,0.024750399589538574
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,15,0.02675839960575104
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,15,0.024694399535655977
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,31,0.026743999123573302
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,31,0.024736000597476958
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,63,0.02680320143699646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,63,0.024646399915218352
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,127,0.03033280074596405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,127,0.024726399779319765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,255,0.04625920057296753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,255,0.03141599893569946
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,511,0.08204320073127747
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,511,0.0488400012254715
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,1023,0.15302239656448363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,131071,0.05014560222625732
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,1023,0.08425279855728149
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,2047,0.15482560396194459
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,2047,0.2942784070968628
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,1,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,1,0.008539199829101562
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,3,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,7,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,15,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,7,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,15,0.008979199826717377
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,31,0.010344000160694122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,31,0.009545599669218063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,63,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,63,0.010480000078678131
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,127,0.010311999917030334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,255,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,255,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,511,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,511,0.010526400059461594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,1023,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,1023,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,2047,0.01077599972486496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,2047,0.010500799864530563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,4095,0.012574400007724761
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,4095,0.012462399899959564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,127,0.01252480000257492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,8191,0.014521600306034088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,8191,0.012619200348854064
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,16383,0.03097440004348755
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,16383,0.016439999639987945
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,32767,0.04796639978885651
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,32767,0.030878400802612303
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,65535,0.048854398727416995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,131071,0.15453120470046997
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,3,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,131071,0.08458719849586487
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,1,0.022961600124835967
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,3,0.02290080040693283
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,1,0.022700800001621245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,3,0.02284799963235855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,7,0.0227183997631073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,127,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,7,0.022860799729824067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,15,0.022710399329662324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,31,0.022870400547981264
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,31,0.022710399329662324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,63,0.02288320064544678
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,63,0.022720000147819518
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,127,0.022862400114536285
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,127,0.02271360009908676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,255,0.022788800299167633
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,255,0.022705599665641785
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,511,0.018692800402641298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,1023,0.020667199790477753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,1023,0.019843199849128725
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,2047,0.02478879988193512
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,2047,0.02271360009908676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,65535,0.08448640108108521
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,1,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,3,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,1,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,7,0.010625600069761276
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,3,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,7,0.010856000334024429
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,15,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,15,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,31,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,15,0.022776000201702118
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,63,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,63,0.009028799831867218
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,127,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,127,0.009895999729633332
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,255,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,255,0.00860000029206276
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,511,0.012356799840927125
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,511,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,1023,0.01454399973154068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,511,0.018739199638366698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,1023,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,2047,0.029892799258232117
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,2047,0.01653600037097931
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,1,0.012691199779510498
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,1,0.012520000338554382
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,3,0.012801599502563477
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,3,0.01252640038728714
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,7,0.012695999443531036
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,7,0.012486399710178375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,15,0.012542399764060973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,15,0.012481600046157837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,31,0.01255359947681427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,31,0.012545600533485413
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,63,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,127,0.01255040019750595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,127,0.012518399953842163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,255,0.0124719999730587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,31,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,511,0.01658879965543747
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,511,0.014659200608730317
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,1023,0.030780801177024843
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,1023,0.019043199717998505
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,2047,0.04891360104084015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,2047,0.03215680122375488
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,1,0.014606399834156037
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,63,0.012849600613117218
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,3,0.01446560025215149
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,1,0.012702399492263794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,7,0.014670400321483612
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,3,0.01260959953069687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,7,0.012628799676895142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,255,0.012468799948692322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,15,0.014657600224018097
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,15,0.012563200294971466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,31,0.012588800489902496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,31,0.014504000544548035
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,63,0.012654399871826172
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,255,0.014638400077819825
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,127,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,127,0.014502400159835815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,255,0.012531200051307678
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,511,0.029257598519325256
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,511,0.016468800604343414
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,1023,0.0488783985376358
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,1023,0.030888000130653383
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,2047,0.08568000197410583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,2047,0.05069440007209778
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,1,0.0207056000828743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,3,0.02268480062484741
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,3,0.020657600462436677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,7,0.022672000527381896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,7,0.020692799985408784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,15,0.022750400006771088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,15,0.020659199357032774
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,31,0.022737599909305573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,31,0.020683200657367708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,63,0.022622400522232057
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,63,0.020737600326538087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,127,0.022753599286079406
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,127,0.02065120041370392
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,255,0.021132799983024596
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,255,0.020652799308300017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,511,0.016579200327396394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,1023,0.014496000111103058
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,1023,0.01451520025730133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,2047,0.014524799585342408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,2047,0.014449599385261535
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,1,0.014606399834156037
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,1,0.014612799882888794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,3,0.014571200311183929
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,3,0.014511999487876893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,7,0.014609600603580474
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,7,0.014535999298095703
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,15,0.014611199498176575
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,15,0.014539200067520141
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,31,0.014609600603580474
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,31,0.014595200121402741
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,63,0.014547200500965118
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,1,0.02216479927301407
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,63,0.014521600306034088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,127,0.016540800034999848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,127,0.014553600549697876
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,255,0.016603200137615202
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,255,0.01454399973154068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,511,0.014556799829006196
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,511,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,1023,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,1023,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,2047,0.01244639977812767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,2047,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,1,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,3,0.012353599816560746
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,511,0.01847359985113144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,3,0.011671999841928482
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,7,0.010553599894046783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,7,0.012491200119256973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,15,0.01263200044631958
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,31,0.012535999715328216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,63,0.01465120017528534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,31,0.012444800138473511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,63,0.012513600289821625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,63,0.012399999797344208
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,127,0.012387199699878693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,127,0.01242400035262108
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,255,0.012511999905109405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,255,0.012441600114107132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,511,0.01242239996790886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,511,0.01069599986076355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,1023,0.010555200278759003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,1,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,1023,0.0105103999376297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,2047,0.011497599631547928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,2047,0.012511999905109405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,1,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,1,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,3,0.010494399815797806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,3,0.010574399679899215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,7,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,7,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,15,0.012479999661445617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,15,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,31,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,31,0.010489600151777268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,63,0.010499200224876404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,63,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,127,0.010513599961996078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,127,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,255,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,255,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,511,0.01056319996714592
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,511,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,1023,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,2047,0.010964799672365189
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,2047,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,1,0.022729599475860597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,1,0.020817600190639496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,3,0.022720000147819518
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,3,0.020732800662517547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,7,0.022745600342750548
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,7,0.02271520048379898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,15,0.022724799811840057
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,15,0.022569599747657775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,31,0.02269120067358017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,31,0.022703999280929567
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,63,0.02271520048379898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,63,0.021911999583244322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,127,0.02294880002737045
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,127,0.02070239931344986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,255,0.022655999660491942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,255,0.020659199357032774
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,15,0.010497599840164185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,511,0.0186831995844841
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,511,0.01666879951953888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,1023,0.014612799882888794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,2047,0.014504000544548035
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,1,0.016516800224781036
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,1,0.016465599834918975
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,3,0.016539199650287627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,1023,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,3,0.016446399688720702
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,7,0.016475200653076172
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,7,0.016487999260425566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,15,0.015335999429225922
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,15,0.01650400012731552
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,31,0.015719999372959138
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,31,0.01608320027589798
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,63,0.016624000668525696
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,127,0.01663679927587509
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,127,0.016612799465656282
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,255,0.01647839993238449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,255,0.01576800048351288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,511,0.014364799857139588
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,511,0.01451520025730133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,1023,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,1023,0.012345600128173827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,2047,0.012585599720478059
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,2047,0.012404800206422806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,1,0.012563200294971466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,1,0.012491200119256973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,3,0.01257600039243698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,3,0.012383999675512314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,1023,0.014779199659824372
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,7,0.012444800138473511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,15,0.012579199671745301
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,15,0.012390399724245072
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,31,0.012563200294971466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,31,0.012439999729394913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,63,0.012529599666595458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,63,0.012537600100040435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,127,0.012585599720478059
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,63,0.016169600188732147
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,127,0.012432000041007996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,255,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,255,0.012415999919176102
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,511,0.010998400300741196
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,511,0.01056319996714592
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,1023,0.010502400249242783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,1023,0.01056160032749176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,2047,0.012465599924325943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,2047,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,1,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,1,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,3,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,3,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,2047,0.014539200067520141
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,7,0.010569600015878677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,15,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,7,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,15,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,31,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,31,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,63,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,63,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,127,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,127,0.010468800365924836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,255,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,255,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,511,0.010470400005578995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,511,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,1023,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,1023,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,2047,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,2047,0.012292800098657608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,1,0.022711999714374542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,3,0.02269120067358017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,3,0.022812800109386445
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,7,0.023151999711990355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,7,0.02272160053253174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,15,0.022697600722312927
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,15,0.022745600342750548
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,31,0.022705599665641785
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,31,0.0227743998169899
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,63,0.022808000445365906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,63,0.02268960028886795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,127,0.022724799811840057
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,7,0.012563200294971466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,127,0.022752000391483305
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,255,0.022788800299167633
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,255,0.022758400440216063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,511,0.018644799292087556
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,511,0.018638400733470915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,1023,0.016491200029850005
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,1023,0.01658560037612915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,2047,0.018676799535751343
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,2047,0.01658719927072525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,1,0.016710400581359863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,1,0.01655679941177368
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,3,0.01674720048904419
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,3,0.016494399309158324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,7,0.016726399958133697
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,15,0.01653759926557541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,15,0.016686399281024934
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,7,0.016956800222396852
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,31,0.01655679941177368
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,31,0.016678400337696075
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,1,0.022708800435066224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,63,0.016774399578571318
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,63,0.016575999557971954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,127,0.016676799952983858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,255,0.016612799465656282
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,255,0.01655520051717758
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,511,0.014505599439144135
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,1023,0.016543999314308167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,1023,0.014777599275112152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,2047,0.018644799292087556
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,2047,0.018535999953746794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,1,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,1,0.010499200224876404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,3,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,3,0.010503999888896942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,7,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,7,0.010320000350475311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,15,0.0104592002928257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,15,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,31,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,31,0.010342399775981902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,63,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,63,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,127,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,127,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,255,0.010476800054311753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,255,0.010375999659299851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,511,0.011528000235557556
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,511,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,1023,0.014446400105953217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,1023,0.012564800679683685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,2047,0.029151999950408937
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,2047,0.016663999855518342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,1,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,1,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,3,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,3,0.01058880016207695
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,7,0.010500799864530563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,7,0.010521599650382995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,15,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,15,0.010532800108194351
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,31,0.010726399719715118
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,31,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,63,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,63,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,127,0.010480000078678131
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,127,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,127,0.01653600037097931
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,511,0.01459839940071106
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,255,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,511,0.013492800295352936
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,511,0.012503999471664428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,1023,0.02890239953994751
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,1023,0.014542399346828461
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,2047,0.04715040028095245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,2047,0.03017440140247345
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,1,0.022710399329662324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,1,0.022731199860572815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,3,0.022756800055503845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,3,0.022649599611759184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,7,0.02274399995803833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,7,0.02280319929122925
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,15,0.02271520048379898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,15,0.022697600722312927
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,31,0.02284960001707077
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,31,0.02271360009908676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,63,0.022745600342750548
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,63,0.022700800001621245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,127,0.022679999470710754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,127,0.02083200067281723
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,255,0.022648000717163087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,255,0.020681600272655486
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,511,0.018500800430774688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,511,0.01658080071210861
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,1023,0.01451839953660965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,2047,0.014575999975204468
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,2047,0.014510400593280792
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,1,0.016257600486278535
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,1,0.014632000029087067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,3,0.016579200327396394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,3,0.01656319946050644
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,255,0.010521599650382995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,7,0.01656319946050644
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,15,0.016572800278663636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,15,0.01652960032224655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,31,0.015928000211715698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,31,0.01566080003976822
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,63,0.016550399363040924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,63,0.01552480012178421
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,127,0.016396799683570863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,127,0.01475680023431778
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,255,0.014619199931621552
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,255,0.016641600430011748
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,511,0.014550399780273438
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,511,0.014537599682807923
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,1023,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,2047,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,2047,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,1023,0.014587199687957764
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,1,0.01249760016798973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,1,0.012479999661445617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,3,0.012489599734544754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,3,0.01252480000257492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,7,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,15,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,15,0.012486399710178375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,7,0.01656000018119812
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,31,0.012468799948692322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,31,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,63,0.012535999715328216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,127,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,127,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,255,0.012462399899959564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,255,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,511,0.012129600346088409
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,511,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,1023,0.0104592002928257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,1023,0.01064160019159317
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,2047,0.012435200065374375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,2047,0.012443199753761292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,1,0.0105103999376297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,1023,0.012604799866676331
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,1,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,3,0.01053759977221489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,3,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,7,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,15,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,15,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,31,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,31,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,63,0.010480000078678131
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,7,0.012585599720478059
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,63,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,127,0.010513599961996078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,127,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,255,0.010579200088977813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,255,0.010358399897813796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,511,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,511,0.009385599941015243
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,63,0.012547199428081513
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,1023,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,1023,0.010558400303125381
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,2047,0.010476800054311753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,2047,0.010603199899196624
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,1,0.012564800679683685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,3,0.012470400333404541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,1,0.010529600083827972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,3,0.010556799918413162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,7,0.012451200187206269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,7,0.01056160032749176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,15,0.012460800260305405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,15,0.010566399991512298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,31,0.01250080019235611
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,31,0.01061440035700798
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,63,0.010576000064611435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,127,0.012380799651145935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,127,0.010628800094127654
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,255,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,7,0.010547199845314026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,255,0.010691200196743012
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,511,0.014430400729179383
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,511,0.012574400007724761
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,1023,0.016708800196647645
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,1023,0.01648000031709671
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,1,0.012590399384498597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,1,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,3,0.012675200402736665
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,7,0.012559999525547028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,3,0.01242239996790886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,7,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,15,0.01252480000257492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,31,0.012535999715328216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,31,0.012414400279521943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,63,0.012567999958992004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,63,0.012385600060224534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,127,0.012571200728416443
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,127,0.012414400279521943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,255,0.012692800164222718
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,255,0.012137600034475327
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,511,0.016582399606704712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,511,0.01462399959564209
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,63,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,1023,0.03112959861755371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,1023,0.018649600446224213
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,1,0.01873439997434616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,3,0.01873439997434616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,1,0.0165583997964859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,3,0.016784000396728515
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,7,0.01866399943828583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,15,0.018700799345970152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,15,0.016622400283813475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,31,0.018699200451374055
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,31,0.01656160056591034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,63,0.018620799481868743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,63,0.016612799465656282
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,15,0.012615999579429627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,127,0.018628799915313722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,255,0.018855999410152435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,127,0.01701280027627945
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,255,0.01671999990940094
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,511,0.03234080076217651
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,511,0.021454399824142455
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,1023,0.05191680192947388
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,1,0.020662400126457214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,1023,0.033267199993133545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,1,0.01658560037612915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,3,0.0168272003531456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,7,0.024430400133132933
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,7,0.016790400445461272
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,15,0.02072480022907257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,15,0.01666560024023056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,31,0.020787200331687926
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,31,0.016624000668525696
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,63,0.02070239931344986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,63,0.01656640022993088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,127,0.021031999588012697
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,127,0.01653600037097931
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,255,0.029470399022102356
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,255,0.0166703999042511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,511,0.04800800085067749
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,511,0.030720001459121703
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,1023,0.085479998588562
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,1023,0.04984320104122162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,1,0.016579200327396394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,3,0.01658080071210861
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,3,0.014528000354766845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,7,0.016700799763202667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,7,0.014588800072669984
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,15,0.016548800468444824
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,15,0.014560000598430633
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,31,0.01653600037097931
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,31,0.014678399264812469
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,63,0.01650400012731552
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,63,0.014595200121402741
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,127,0.016568000614643096
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,127,0.014601600170135499
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,255,0.016551999747753142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,255,0.014542399346828461
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,1,0.017827199399471284
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,1,0.014664000272750855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,3,0.01671359986066818
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,3,0.014688000082969666
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,7,0.01693439930677414
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,1,0.014579200744628906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,7,0.0147024005651474
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,7,0.016595199704170227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,15,0.01655679941177368
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,15,0.014529600739479065
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,31,0.016595199704170227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,31,0.014579200744628906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,63,0.01655520051717758
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,63,0.014638400077819825
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,127,0.016843199729919434
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,127,0.014542399346828461
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,255,0.018572799861431122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,255,0.014627200365066529
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,1,0.02916640043258667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,1,0.02686559855937958
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,3,0.028935998678207397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,3,0.026840001344680786
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,7,0.028934401273727418
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,7,0.026875200867652892
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,15,0.028881600499153136
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,15,0.026870399713516235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,31,0.02885119915008545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,3,0.020795199275016784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,63,0.028918400406837463
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,63,0.026841598749160766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,127,0.030820798873901368
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,127,0.02688319981098175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,255,0.0343968003988266
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,255,0.028415998816490172
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,1,0.03495360016822815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,1,0.02678079903125763
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,3,0.034995201230049136
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,3,0.026740801334381104
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,7,0.034999999403953555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,7,0.026697599887847902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,15,0.03502880036830902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,15,0.026862400770187377
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,31,0.03502399921417236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,31,0.02672159969806671
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,63,0.035016000270843506
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,63,0.026814401149749756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,127,0.03697119951248169
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,127,0.02675040066242218
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,255,0.0489520013332367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,255,0.03164480030536652
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,1,0.022771200537681578
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,1,0.020750400424003602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,3,0.022780799865722658
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,3,0.020768000185489653
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,7,0.022785599529743194
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,7,0.020716799795627593
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,15,0.0227183997631073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,15,0.020817600190639496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,31,0.02274399995803833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,31,0.0208079993724823
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,63,0.022676800191402436
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,63,0.02083040028810501
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,127,0.02301120012998581
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,127,0.02077919989824295
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,1,0.024958400428295134
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,1,0.022782400250434875
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,3,0.025150400400161744
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,31,0.026843199133872987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,3,0.022648000717163087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,7,0.024828800559043886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,15,0.02492000013589859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,15,0.0220768004655838
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,31,0.022567999362945557
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,31,0.02597759962081909
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,63,0.025003200769424437
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,63,0.02265920042991638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,127,0.02741599977016449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,1,0.051507198810577394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,1,0.04615679979324341
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,3,0.045721599459648134
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,7,0.05163360238075256
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,7,0.04558559954166412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,15,0.05154079794883728
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,15,0.04548960030078888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,31,0.05188959836959839
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,31,0.04539999961853027
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,63,0.05437279939651489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,63,0.04545120000839233
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,127,0.0580128014087677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,127,0.048153600096702574
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,1,0.061868798732757566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,1,0.045377600193023684
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,7,0.022711999714374542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,3,0.061776000261306765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,3,0.04536640048027039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,7,0.061806398630142215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,7,0.044964799284935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,127,0.02208160012960434
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,15,0.06172800064086914
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,15,0.044412800669670106
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,31,0.0631983995437622
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,3,0.05143839716911316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,31,0.04530879855155945
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,63,0.06371200084686279
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,63,0.045179200172424314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,127,0.04684799909591675
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,1,0.022750400006771088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,3,0.02292799949645996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,3,0.02271520048379898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,7,0.02297919988632202
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,7,0.022711999714374542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,15,0.02285120040178299
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,15,0.022865599393844603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,31,0.0226623997092247
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,31,0.02268960028886795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,63,0.02285120040178299
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,63,0.022681599855422972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,127,0.02274720072746277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,127,0.022711999714374542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,255,0.02268799990415573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,255,0.022265599668025972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,511,0.018607999384403228
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,511,0.018671999871730804
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,1023,0.016648000478744505
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,1023,0.014753599464893342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,2047,0.015216000378131866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,127,0.06371200084686279
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,2047,0.01465120017528534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,1,0.016684800386428833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,1,0.016550399363040924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,3,0.016510400176048278
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,7,0.01650400012731552
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,7,0.016599999368190767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,15,0.016684800386428833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,15,0.016579200327396394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,31,0.01666239947080612
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,31,0.01656640022993088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,63,0.016523200273513793
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,63,0.016484799981117248
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,127,0.0165120005607605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,127,0.016582399606704712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,255,0.01669439971446991
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,255,0.016510400176048278
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,511,0.01454399973154068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,511,0.0144896000623703
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,1023,0.01260959953069687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,1023,0.01266079992055893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,2047,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,1,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,1,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,3,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,1,0.022735999524593355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,3,0.012392000108957291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,7,0.012303999811410903
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,7,0.012443199753761292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,15,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,15,0.012521600723266602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,31,0.010576000064611435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,31,0.012539200484752655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,63,0.010622400045394897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,63,0.012350399792194367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,127,0.011723200231790543
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,127,0.011430399864912033
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,255,0.010499200224876404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,255,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,511,0.010480000078678131
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,511,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,1023,0.010523200035095215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,1023,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,2047,0.01249760016798973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,2047,0.010582400113344192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,1,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,1,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,3,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,3,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,7,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,7,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,15,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,15,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,31,0.010476800054311753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,31,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,63,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,63,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,127,0.010476800054311753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,255,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,127,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,255,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,511,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,511,0.010355199873447418
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,1023,0.010849600285291671
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,1023,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,2047,0.0125231996178627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,2047,0.012569600343704223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,2047,0.01244639977812767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,1,0.03509120047092438
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,1,0.031017601490020752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,3,0.03499679863452911
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,3,0.030961599946022034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,7,0.0350816011428833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,7,0.030902400612831116
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,15,0.030972799658775328
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,31,0.03672640025615692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,31,0.031009599566459656
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,63,0.03712159991264343
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,63,0.031060799956321716
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,1,0.04318400025367737
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,3,0.016553600132465363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,1,0.03699040114879608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,3,0.04312480092048645
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,3,0.03705439865589142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,7,0.04317600131034851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,7,0.037049600481987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,15,0.0431439995765686
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,15,0.03704479932785034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,31,0.04357919991016388
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,31,0.03501920104026794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,63,0.045526400208473206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,63,0.03518719971179962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,1,0.09652000069618225
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,1,0.08415359854698182
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,3,0.09656479954719543
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,3,0.08405280113220215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,7,0.09690399765968323
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,7,0.08413599729537964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,15,0.09864799976348877
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,15,0.08417279720306396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,31,0.09999039769172668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,31,0.0844655990600586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,63,0.10171680450439453
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,63,0.08673440217971802
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,1,0.1169808030128479
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,1,0.08009600043296813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,3,0.1169935941696167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,3,0.08008639812469483
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,15,0.03520799875259399
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,7,0.11711200475692748
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,7,0.08016960024833679
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,15,0.11716959476470948
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,15,0.08207839727401733
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,31,0.11685600280761718
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,31,0.08218079805374146
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,63,0.11720000505447388
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,63,0.08218560218811036
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,1,0.06577119827270508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,1,0.054287999868392944
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,3,0.06534079909324646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,3,0.053592002391815184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,7,0.06580640077590942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,7,0.05352479815483093
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,15,0.06547200083732604
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,15,0.055491197109222415
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,31,0.06533920168876647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,31,0.05560160279273987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,1,0.07806400060653687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,1,0.06376479864120484
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,3,0.07812479734420777
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,3,0.06385120153427123
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,7,0.07809759974479676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,7,0.06453760266304016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,15,0.0784704029560089
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,15,0.06571360230445862
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,31,0.07886880040168762
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,31,0.06569920182228088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,1,0.186518394947052
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,3,0.18744479417800902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,1,0.1621440052986145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,3,0.16098560094833375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,7,0.18722399473190307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,15,0.18851360082626342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,15,0.1623296022415161
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,31,0.19089119434356688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,31,0.16234079599380494
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,1,0.15386879444122314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,3,0.15394239425659179
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,7,0.2258591890335083
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,7,0.1539039969444275
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,15,0.22635838985443116
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,15,0.15445120334625245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,31,0.22623679637908936
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,31,0.15413279533386232
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,1,0.022801600396633148
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,1,0.022708800435066224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,7,0.1623520016670227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,3,0.022752000391483305
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,7,0.022755199670791627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,7,0.022728000581264497
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,15,0.02274720072746277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,15,0.022758400440216063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,1,0.22557439804077148
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,31,0.02269279956817627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,31,0.022707200050354003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,63,0.022720000147819518
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,3,0.22560639381408693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,127,0.022703999280929567
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,127,0.02259040027856827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,255,0.02269279956817627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,255,0.022598400712013245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,511,0.018643200397491455
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,511,0.01849920004606247
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,1023,0.014633600413799287
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,2047,0.01648000031709671
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,2047,0.01462240070104599
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,1,0.016624000668525696
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,1,0.016542400419712066
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,3,0.016740800440311433
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,3,0.022771200537681578
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,3,0.01659200042486191
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,7,0.016729600727558136
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,7,0.016678400337696075
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,15,0.01668799966573715
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,15,0.016631999611854555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,31,0.016599999368190767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,31,0.016649599373340606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,63,0.022703999280929567
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,63,0.016705599427223206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,63,0.016572800278663636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,127,0.01671359986066818
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,127,0.01661120057106018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,255,0.01669120043516159
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,1023,0.016467200219631196
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,255,0.016625599563121797
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,511,0.014524799585342408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,511,0.014524799585342408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,1023,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,2047,0.014548799395561219
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,1,0.012460800260305405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,1,0.01247360035777092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,3,0.01249760016798973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,3,0.01252480000257492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,7,0.012518399953842163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,7,0.012494400143623352
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,15,0.012542399764060973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,15,0.01252640038728714
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,31,0.012511999905109405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,31,0.01250080019235611
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,63,0.012404800206422806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,63,0.012444800138473511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,127,0.012521600723266602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,127,0.012518399953842163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,255,0.012520000338554382
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,255,0.012489599734544754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,511,0.012004800140857697
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,511,0.010486400127410889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,1023,0.012729600071907043
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,1023,0.012462399899959564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,2047,0.014819200336933135
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,2047,0.013956800103187561
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,1,0.008628799766302108
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,1,0.00835840031504631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,3,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,2047,0.013607999682426453
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,7,0.008913599699735642
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,7,0.008425600081682205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,15,0.009035199880599976
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,15,0.008488000184297562
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,31,0.008708799630403519
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,31,0.008406399935483932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,63,0.009031999856233597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,63,0.008367999643087386
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,127,0.00941760018467903
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,127,0.008395200222730636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,255,0.00921119973063469
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,255,0.008337599784135818
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,511,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,511,0.008851200342178345
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,1023,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,1023,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,2047,0.027316799759864806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,1,0.020609599351882935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,2047,0.014499199390411378
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,1,0.02035519927740097
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,3,0.02062239944934845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,7,0.02066880017518997
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,7,0.020652799308300017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,15,0.02067199945449829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,15,0.02057439982891083
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,31,0.020692799985408784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,31,0.02066880017518997
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,63,0.02067359983921051
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,63,0.020688000321388244
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,127,0.020664000511169435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,127,0.020628799498081208
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,255,0.02066880017518997
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,3,0.00865119993686676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,511,0.016774399578571318
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,511,0.01671359986066818
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,1023,0.01884319931268692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,1023,0.01863359957933426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,2047,0.022737599909305573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,2047,0.02089280039072037
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,1,0.010310400277376175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,1,0.008750399947166443
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,3,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,3,0.009539200365543366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,3,0.020740799605846405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,7,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,7,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,15,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,15,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,31,0.010340800136327743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,31,0.010345599800348281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,63,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,63,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,127,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,127,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,255,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,255,0.01035040020942688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,511,0.012388800084590913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,511,0.010564800351858139
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,1023,0.01451839953660965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,1023,0.012624000012874604
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,2047,0.029569599032402038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,1023,0.01263200044631958
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,2047,0.01648160070180893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,1,0.012611199915409089
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,1,0.012535999715328216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,3,0.012675200402736665
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,3,0.01242239996790886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,7,0.012615999579429627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,7,0.012425599992275238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,15,0.012417600303888322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,31,0.01263200044631958
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,31,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,63,0.012579199671745301
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,63,0.012622399628162384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,127,0.012838399410247803
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,127,0.012540799379348756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,255,0.012544000148773193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,255,0.012532800436019897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,511,0.01648319959640503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,511,0.014604799449443817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,1023,0.030606400966644288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,1023,0.01857440024614334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,2047,0.049353599548339844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,2047,0.03224000036716461
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,1,0.014640000462532044
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,1,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,3,0.014664000272750855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,3,0.012532800436019897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,255,0.020665599405765532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,7,0.014582400023937226
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,7,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,15,0.014521600306034088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,15,0.01247360035777092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,31,0.014595200121402741
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,15,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,63,0.014585599303245544
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,31,0.012494400143623352
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,63,0.012600000202655792
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,127,0.014668799936771393
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,255,0.01472959965467453
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,255,0.01252480000257492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,511,0.029300799965858458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,511,0.016603200137615202
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,1023,0.048607999086380006
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,1023,0.030961599946022034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,2047,0.08560479879379272
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,2047,0.0511680006980896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,1,0.020657600462436677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,1,0.018716800212860107
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,3,0.020713600516319274
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,3,0.01876319944858551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,7,0.02059520035982132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,7,0.018694399297237395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,15,0.018636800348758698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,15,0.018713599443435668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,31,0.019064000248908995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,31,0.01866399943828583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,63,0.020686399936676026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,63,0.018750399351119995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,127,0.018614399433135986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,127,0.018799999356269838
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,255,0.018771199882030486
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,255,0.018587200343608855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,511,0.01645279973745346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,1023,0.01247360035777092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,1023,0.01255359947681427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,2047,0.014523200690746307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,2047,0.01448799967765808
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,1,0.014534400403499603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,1,0.014499199390411378
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,3,0.01448799967765808
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,3,0.014431999623775482
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,7,0.014508800208568573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,15,0.014667199552059173
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,15,0.014528000354766845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,31,0.014614400267601014
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,31,0.014443199336528777
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,63,0.014582400023937226
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,63,0.014529600739479065
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,127,0.01250240057706833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,127,0.014660799503326416
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,255,0.014475199580192565
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,255,0.014459200203418732
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,511,0.012559999525547028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,511,0.012622399628162384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,1023,0.012591999769210816
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,1023,0.012544000148773193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,2047,0.0124208003282547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,2047,0.012438400089740754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,1,0.012515200674533844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,1,0.012404800206422806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,3,0.012307199835777282
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,3,0.012281599640846252
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,511,0.016657599806785585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,7,0.01189119964838028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,7,0.01048159971833229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,15,0.012348800152540206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,15,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,31,0.011796800047159195
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,7,0.014556799829006196
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,31,0.010734400153160096
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,63,0.010543999820947647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,63,0.010542400181293488
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,127,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,127,0.011180800199508668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,255,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,511,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,511,0.010550399869680404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,1023,0.010543999820947647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,1023,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,2047,0.012460800260305405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,2047,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,1,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,1,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,3,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,3,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,7,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,7,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,15,0.010496000200510025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,15,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,31,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,31,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,63,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,63,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,127,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,127,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,255,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,255,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,511,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,255,0.012439999729394913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,511,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,1023,0.01056319996714592
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,1023,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,2047,0.012494400143623352
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,2047,0.010552000254392624
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,1,0.0205935999751091
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,1,0.0186271995306015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,3,0.020614400506019592
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,3,0.018643200397491455
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,7,0.02062560021877289
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,7,0.01860959976911545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,15,0.020609599351882935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,15,0.018648000061511995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,31,0.02064319998025894
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,31,0.01871519982814789
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,63,0.020623999834060668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,127,0.020555199682712556
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,127,0.01879200041294098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,255,0.0187376007437706
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,255,0.01865919977426529
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,511,0.016579200327396394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,511,0.01688160002231598
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,1023,0.01456640064716339
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,1023,0.014467200636863709
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,2047,0.014574399590492249
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,2047,0.014486399292945863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,1,0.01451680064201355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,1,0.014480000734329224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,3,0.014433600008487701
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,3,0.014521600306034088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,7,0.014508800208568573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,7,0.014441600441932679
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,15,0.014460800588130951
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,15,0.01446560025215149
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,31,0.014571200311183929
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,31,0.014427199959754944
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,63,0.014591999351978302
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,63,0.014441600441932679
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,127,0.014550399780273438
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,127,0.014468799531459808
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,255,0.014550399780273438
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,511,0.012756800651550293
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,511,0.012558400630950928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,1023,0.012641599774360657
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,1023,0.012539200484752655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,2047,0.012478400021791458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,127,0.01268479973077774
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,2047,0.012361600250005721
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,1,0.012432000041007996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,1,0.012409599870443344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,3,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,3,0.012409599870443344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,7,0.012544000148773193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,7,0.0106175996363163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,15,0.010476800054311753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,15,0.010864000022411346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,63,0.018612800538539885
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,31,0.010567999631166457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,31,0.011086399853229522
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,63,0.010480000078678131
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,63,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,127,0.012361600250005721
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,127,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,255,0.012425599992275238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,511,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,511,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,1023,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,1023,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,2047,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,2047,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,1,0.010555200278759003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,1,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,3,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,3,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,7,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,7,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,255,0.014475199580192565
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,15,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,31,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,15,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,31,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,63,0.0105103999376297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,127,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,127,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,255,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,255,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,511,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,511,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,1023,0.010470400005578995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,255,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,1023,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,2047,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,2047,0.010499200224876404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,1,0.020657600462436677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,1,0.018654400110244752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,3,0.020683200657367708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,3,0.01887039989233017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,7,0.020689600706100465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,7,0.019120000302791595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,15,0.020662400126457214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,15,0.02056799978017807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,31,0.020718400180339814
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,31,0.020688000321388244
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,63,0.02059199959039688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,63,0.020664000511169435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,127,0.020664000511169435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,127,0.020563200116157532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,255,0.020657600462436677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,255,0.018731200695037843
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,511,0.016655999422073364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,511,0.016572800278663636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,1023,0.014547200500965118
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,1023,0.014505599439144135
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,2047,0.016659200191497803
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,2047,0.016681599617004394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,1,0.014561599493026734
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,63,0.010503999888896942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,3,0.014511999487876893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,3,0.014721600711345673
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,7,0.014531199634075165
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,7,0.014614400267601014
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,15,0.01451839953660965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,15,0.014705599844455719
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,31,0.014734399318695069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,31,0.014731200039386749
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,63,0.014523200690746307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,63,0.014747199416160584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,127,0.014767999947071075
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,127,0.014646400511264802
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,255,0.014654399454593658
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,255,0.014679999649524688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,511,0.014526399970054626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,511,0.012479999661445617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,1023,0.016487999260425566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,1023,0.014584000408649444
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,2047,0.018036800622940063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,2047,0.01663520038127899
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,1,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,3,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,3,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,7,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,7,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,15,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,15,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,31,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,31,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,63,0.01048159971833229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,63,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,127,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,127,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,255,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,255,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,511,0.012371200323104858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,511,0.0104592002928257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,1023,0.01448799967765808
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,1023,0.012511999905109405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,2047,0.029337599873542786
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,2047,0.016596800088882445
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,1,0.012140800058841706
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,1,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,3,0.012110400199890136
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,1,0.014524799585342408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,3,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,7,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,15,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,15,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,31,0.01055999994277954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,31,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,63,0.010595200210809707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,63,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,127,0.01053600013256073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,127,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,255,0.010521599650382995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,255,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,1,0.010552000254392624
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,511,0.013150399923324585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,511,0.012412799894809723
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,1023,0.014697599411010741
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,2047,0.047100800275802615
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,2047,0.02991519868373871
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,1,0.018726399540901183
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,1,0.018713599443435668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,3,0.01908160001039505
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,3,0.018753600120544434
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,7,0.020572799444198608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,7,0.01879359930753708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,15,0.020636799931526183
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,7,0.010710400342941285
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,15,0.018727999925613404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,31,0.020633600652217865
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,31,0.01868479996919632
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,63,0.02054239958524704
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,63,0.019508799910545348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,127,0.02057439982891083
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,127,0.018787199258804323
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,255,0.01977760046720505
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,255,0.018670399487018586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,511,0.01647839993238449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,1023,0.014582400023937226
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,1023,0.01446560025215149
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,2047,0.014596800506114959
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,2047,0.01451680064201355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,1,0.014632000029087067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,1023,0.028830400109291075
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,1,0.014523200690746307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,3,0.014580799639225006
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,3,0.014550399780273438
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,7,0.01449279934167862
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,15,0.014478400349617004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,15,0.014519999921321868
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,31,0.014511999487876893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,31,0.014505599439144135
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,63,0.014627200365066529
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,63,0.014468799531459808
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,127,0.01465280055999756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,127,0.01456640064716339
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,255,0.014472000300884247
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,255,0.014483200013637542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,511,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,1023,0.012438400089740754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,1023,0.012489599734544754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,2047,0.012436799705028534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,2047,0.012462399899959564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,511,0.016710400581359863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,1,0.012428800016641617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,3,0.012486399710178375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,3,0.011316800117492675
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,7,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,7,0.01051200032234192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,15,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,15,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,31,0.012470400333404541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,31,0.012404800206422806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,7,0.014564800262451171
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,63,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,127,0.012465599924325943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,63,0.010531199723482132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,127,0.011627200245857238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,255,0.012465599924325943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,255,0.012444800138473511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,511,0.010499200224876404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,511,0.010516799986362457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,1023,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,1023,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,2047,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,2047,0.01250080019235611
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,1,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,1,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,3,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,3,0.010367999970912933
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,7,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,7,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,15,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,511,0.012559999525547028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,31,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,15,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,31,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,63,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,127,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,127,0.010478399693965912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,255,0.010329599678516387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,255,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,511,0.010344000160694122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,511,0.00957920029759407
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,1023,0.010542400181293488
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,1023,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,2047,0.012414400279521943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,2047,0.010529600083827972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,1,0.012449599802494049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,1,0.010556799918413162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,3,0.01247360035777092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,3,0.010692799836397171
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,7,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,7,0.011049599945545196
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,15,0.01233920007944107
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,15,0.010648000240325927
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,31,0.012392000108957291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,31,0.010547199845314026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,63,0.01231520026922226
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,63,0.010494399815797806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,127,0.012451200187206269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,127,0.01055999994277954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,255,0.012380799651145935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,63,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,255,0.010553599894046783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,511,0.012569600343704223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,511,0.012558400630950928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,1023,0.016622400283813475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,1023,0.01459999978542328
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,1,0.012505599856376648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,1,0.01136159971356392
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,3,0.012508800625801087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,1,0.012452799826860428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,7,0.012467200309038163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,7,0.012432000041007996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,15,0.012544000148773193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,31,0.012566399574279786
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,15,0.012425599992275238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,31,0.012059199810028075
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,63,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,127,0.012507200241088867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,127,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,255,0.012511999905109405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,255,0.010523200035095215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,511,0.016572800278663636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,511,0.012916800379753113
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,1023,0.030428799986839294
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,1023,0.018654400110244752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,1,0.01863519996404648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,1,0.016883200407028197
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,3,0.018624000251293182
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,7,0.018646399676799773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,7,0.016575999557971954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,15,0.01860480010509491
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,15,0.016599999368190767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,31,0.018632000684738158
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,31,0.016590400040149687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,63,0.018756799399852753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,3,0.01141280010342598
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,63,0.016648000478744505
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,127,0.018743999302387238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,127,0.016657599806785585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,255,0.01876160055398941
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,255,0.016667200624942778
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,511,0.03243519961833954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,511,0.020670400559902193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,63,0.012567999958992004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,1023,0.051793599128723146
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,1023,0.03317759931087494
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,1,0.021566399931907655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,1,0.017659200727939604
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,3,0.02075680047273636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,3,0.01706880033016205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,7,0.021724799275398256
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,7,0.016595199704170227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,15,0.02088160067796707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,15,0.017502400279045104
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,31,0.016702400147914888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,63,0.021988800168037413
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,63,0.01666560024023056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,127,0.02258560061454773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,127,0.016704000532627106
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,255,0.029385599493980407
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,255,0.017044800519943237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,511,0.04815039932727814
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,1023,0.08434240221977234
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,1023,0.04973919987678528
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,3,0.017003199458122252
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,1,0.014632000029087067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,1,0.014523200690746307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,3,0.01467359960079193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,3,0.014577600359916686
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,7,0.014511999487876893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,15,0.016395199298858642
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,15,0.014545600116252898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,31,0.014608000218868256
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,31,0.014499199390411378
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,63,0.014611199498176575
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,63,0.014523200690746307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,127,0.015545600652694702
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,127,0.014534400403499603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,255,0.014681600034236908
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,255,0.014507199823856353
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,1,0.016553600132465363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,31,0.020684799551963805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,1,0.014507199823856353
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,3,0.016564799845218657
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,3,0.014534400403499603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,511,0.03078399896621704
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,7,0.016572800278663636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,15,0.016648000478744505
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,15,0.014523200690746307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,31,0.01661120057106018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,31,0.014531199634075165
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,63,0.01658399999141693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,63,0.014596800506114959
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,127,0.01659200042486191
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,127,0.014537599682807923
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,255,0.018606400489807128
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,255,0.014499199390411378
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,1,0.028881600499153136
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,1,0.026822400093078614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,3,0.028907200694084166
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,3,0.026824000477790832
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,7,0.028865599632263185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,7,0.026800000667572023
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,15,0.028860801458358766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,15,0.026851201057434083
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,31,0.028907200694084166
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,31,0.026752001047134398
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,63,0.028892800211906433
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,7,0.014643199741840363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,63,0.026820799708366393
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,127,0.029068800806999206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,127,0.02682879865169525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,255,0.03312479853630066
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,255,0.026791998744010927
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,1,0.0350847989320755
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,1,0.02662239968776703
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,7,0.014681600034236908
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,3,0.026561599969863892
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,7,0.0350383996963501
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,7,0.02678079903125763
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,15,0.0350816011428833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,15,0.026767998933792114
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,31,0.035104000568389894
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,31,0.02685439884662628
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,63,0.03512639999389648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,127,0.037049600481987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,63,0.026931199431419372
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,127,0.02680160105228424
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,255,0.04902400076389313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,1,0.022782400250434875
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,1,0.020688000321388244
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,3,0.022703999280929567
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,3,0.020659199357032774
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,7,0.022732800245285033
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,7,0.020633600652217865
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,15,0.02282080054283142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,15,0.02008800059556961
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,31,0.022814400494098663
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,31,0.0205935999751091
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,63,0.022761599719524385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,63,0.019968000054359437
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,127,0.02268799990415573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,127,0.02016959935426712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,1,0.0248416006565094
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,1,0.02075839936733246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,3,0.024803200364112855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,3,0.02064799964427948
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,7,0.02483679950237274
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,7,0.020683200657367708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,15,0.024961599707603456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,15,0.020665599405765532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,31,0.02479359954595566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,31,0.020710399746894835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,63,0.025028800964355467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,63,0.02072480022907257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,127,0.025409600138664244
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,127,0.020689600706100465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,1,0.05127999782562256
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,1,0.04524799883365631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,3,0.05139679908752441
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,3,0.035123199224472046
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,3,0.0452320009469986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,7,0.05137919783592224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,7,0.04538719952106476
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,15,0.05046719908714294
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,15,0.04525119960308075
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,31,0.05135840177536011
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,31,0.04524959921836853
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,63,0.05286880135536194
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,63,0.04532000124454498
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,127,0.055307197570800784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,1,0.06363999843597412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,127,0.046158400177955625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,1,0.04374560117721558
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,3,0.06373599767684937
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,3,0.043275201320648195
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,7,0.0637503981590271
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,255,0.031201601028442383
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,15,0.06366080045700073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,15,0.043433600664138795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,31,0.06380159854888916
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,31,0.044696000218391416
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,63,0.06434400081634521
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,63,0.04519839882850647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,127,0.06427199840545654
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,127,0.0462336003780365
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,1,0.02067999988794327
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,1,0.0187376007437706
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,3,0.020686399936676026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,3,0.018756799399852753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,7,0.020558400452136992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,7,0.018755200505256652
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,15,0.01879200041294098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,15,0.018723200261592864
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,31,0.018675200641155243
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,63,0.020555199682712556
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,63,0.018750399351119995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,127,0.020630399882793426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,127,0.018743999302387238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,255,0.01870400011539459
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,255,0.018673600256443025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,511,0.01658719927072525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,511,0.016540800034999848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,1023,0.014616000652313232
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,1023,0.014502400159835815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,2047,0.01464959979057312
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,2047,0.01467680037021637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,1,0.01462559998035431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,7,0.04350239932537079
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,1,0.014531199634075165
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,3,0.014611199498176575
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,3,0.014550399780273438
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,7,0.014644800126552582
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,7,0.014591999351978302
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,15,0.014681600034236908
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,15,0.01456640064716339
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,31,0.014603200554847717
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,31,0.014604799449443817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,63,0.014604799449443817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,63,0.01456640064716339
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,127,0.014604799449443817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,127,0.01464959979057312
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,255,0.014627200365066529
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,255,0.014604799449443817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,511,0.012617599964141846
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,511,0.012611199915409089
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,1023,0.012537600100040435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,31,0.020294399559497835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,1023,0.012489599734544754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,2047,0.012548799812793731
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,1,0.010580799728631973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,2047,0.012531200051307678
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,1,0.010523200035095215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,3,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,7,0.010521599650382995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,7,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,15,0.01061599999666214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,15,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,31,0.010572800040245056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,31,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,63,0.010567999631166457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,63,0.010502400249242783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,127,0.011615999788045884
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,127,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,255,0.01109120026230812
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,255,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,511,0.010603199899196624
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,1023,0.010468800365924836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,1023,0.010529600083827972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,2047,0.012379200011491776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,2047,0.0124208003282547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,1,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,1,0.010503999888896942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,3,0.010470400005578995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,3,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,7,0.01048320010304451
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,7,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,15,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,31,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,31,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,63,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,63,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,127,0.010499200224876404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,127,0.0104592002928257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,255,0.01051200032234192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,255,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,511,0.01093439981341362
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,511,0.010497599840164185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,1023,0.012404800206422806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,3,0.010579200088977813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,1023,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,2047,0.012745599448680877
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,2047,0.012513600289821625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,1,0.03503519892692566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,1,0.02919999957084656
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,3,0.035132798552513125
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,3,0.02927680015563965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,7,0.03503200113773346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,7,0.03094559907913208
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,15,0.03504959940910339
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,511,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,31,0.0350383996963501
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,31,0.02977440059185028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,63,0.035124799609184264
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,63,0.02969760000705719
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,1,0.04219360053539276
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,15,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,1,0.035076799988746646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,3,0.04283519983291626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,3,0.03516800105571747
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,7,0.04249599874019623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,7,0.035051199793815616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,15,0.041731199622154234
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,15,0.035158398747444156
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,31,0.043198400735855104
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,31,0.03510079979896545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,63,0.044207999110221864
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,63,0.03502880036830902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,1,0.09452959895133972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,1,0.0828831970691681
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,3,0.09446560144424439
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,3,0.08264319896697998
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,7,0.09446560144424439
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,7,0.08403199911117554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,15,0.09592000246047974
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,15,0.08380640149116517
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,31,0.09738399982452392
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,31,0.08392639756202698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,63,0.09862080216407776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,63,0.08455680012702942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,1,0.11901600360870361
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,15,0.030633598566055298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,3,0.11907520294189453
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,3,0.08013920187950134
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,7,0.11904640197753906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,7,0.08007680177688599
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,15,0.11929440498352051
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,15,0.0801472008228302
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,31,0.11942559480667114
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,63,0.12055039405822754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,63,0.08201760053634644
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,1,0.061603200435638425
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,1,0.04934720098972321
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,3,0.06163039803504944
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,3,0.05034719705581665
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,7,0.061647999286651614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,7,0.051102399826049805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,15,0.06170240044593811
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,15,0.050382399559021
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,31,0.06210079789161682
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,31,0.05097280144691467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,1,0.07746400237083435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,1,0.0800320029258728
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,1,0.061740797758102414
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,3,0.07774400115013122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,3,0.06164640188217163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,7,0.07623680233955384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,7,0.06172320246696472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,15,0.07645120024681092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,15,0.06190559864044189
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,31,0.07699360251426697
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,31,0.06363679766654969
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,1,0.15992159843444825
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,31,0.08210880160331727
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,3,0.18348319530487062
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,3,0.15992480516433716
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,7,0.1595296025276184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,7,0.18442399501800538
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,15,0.1848304033279419
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,15,0.15986239910125732
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,31,0.18538880348205566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,31,0.16155840158462526
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,1,0.15217759609222412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,3,0.23092799186706542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,3,0.15374239683151245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,7,0.15377440452575683
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,7,0.230513596534729
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,15,0.15368800163269042
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,15,0.22980160713195802
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,31,0.23153440952301024
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,1,0.020611199736595153
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,1,0.01863359957933426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,3,0.020684799551963805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,3,0.018694399297237395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,7,0.020694400370121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,7,0.018622399866580965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,15,0.0186831995844841
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,15,0.01873279958963394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,1,0.1832319974899292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,31,0.018830400705337525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,31,0.019100800156593323
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,63,0.018854400515556334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,127,0.019755199551582336
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,127,0.01892800033092499
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,255,0.01886879950761795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,1,0.22973439693450928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,255,0.018803200125694274
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,511,0.016788800060749055
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,1023,0.014640000462532044
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,511,0.016675199568271636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,1023,0.01454080045223236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,2047,0.015721599757671356
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,2047,0.014640000462532044
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,1,0.014657600224018097
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,1,0.014667199552059173
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,3,0.014726400375366211
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,31,0.15385600328445434
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,7,0.014595200121402741
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,7,0.014575999975204468
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,15,0.014659200608730317
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,15,0.014688000082969666
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,31,0.01472959965467453
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,31,0.014720000326633453
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,63,0.014571200311183929
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,63,0.01467680037021637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,127,0.014664000272750855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,63,0.018729600310325622
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,127,0.014630399644374847
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,255,0.014579200744628906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,255,0.014710399508476257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,511,0.012539200484752655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,511,0.012625600397586822
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,1023,0.012515200674533844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,1023,0.012465599924325943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,2047,0.014584000408649444
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,2047,0.014399999380111694
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,1,0.01122559979557991
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,3,0.012577599287033081
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,3,0.012481600046157837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,7,0.0124719999730587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,7,0.01231520026922226
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,15,0.012415999919176102
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,15,0.012043199688196182
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,31,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,31,0.012318400293588638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,3,0.014638400077819825
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,63,0.012505599856376648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,63,0.010574399679899215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,127,0.012441600114107132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,127,0.010518400371074677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,255,0.012411200255155564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,255,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,511,0.010576000064611435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,511,0.010523200035095215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,1023,0.012664000689983367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,2047,0.014616000652313232
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,2047,0.014531199634075165
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,1,0.008544000238180161
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,1,0.008441600203514098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,3,0.008508799970149994
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,3,0.008532799780368805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,1,0.011137600243091583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,7,0.008366400003433227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,15,0.009692800045013428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,31,0.009799999743700027
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,31,0.008511999994516373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,63,0.009507200121879578
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,63,0.008595199882984161
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,127,0.008507200330495835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,127,0.008476799726486206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,255,0.008747199922800064
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,255,0.008502399921417237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,511,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,511,0.010289599746465683
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,1023,0.012403199821710587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,1023,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,1023,0.012638400495052337
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,2047,0.02831999957561493
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,2047,0.01459999978542328
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,1,0.016686399281024934
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,1,0.016711999475955964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,3,0.017635199427604675
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,3,0.016711999475955964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,7,0.016705599427223206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,7,0.01663520038127899
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,15,0.01669279932975769
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,15,0.016676799952983858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,31,0.016676799952983858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,63,0.01669279932975769
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,63,0.016652800142765045
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,127,0.01679359972476959
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,7,0.00846719965338707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,255,0.016667200624942778
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,15,0.00836160033941269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,255,0.016633599996566772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,511,0.014511999487876893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,511,0.014502400159835815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,1023,0.016545599699020384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,1023,0.014575999975204468
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,2047,0.01863519996404648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,2047,0.016705599427223206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,4095,0.03326559960842133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,4095,0.020718400180339814
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,1,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,1,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,3,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,7,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,7,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,15,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,15,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,31,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,31,0.01672479957342148
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,31,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,63,0.010622400045394897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,127,0.016678400337696075
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,63,0.010291200131177902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,127,0.010496000200510025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,127,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,255,0.010623999685049058
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,255,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,511,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,511,0.010580799728631973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,1023,0.01456640064716339
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,1023,0.012579199671745301
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,2047,0.016649599373340606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,4095,0.047672000527381894
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,3,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,4095,0.030990400910377504
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,1,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,1,0.010558400303125381
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,3,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,3,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,7,0.010507199913263321
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,7,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,15,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,15,0.01061279997229576
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,31,0.01051200032234192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,31,0.010529600083827972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,63,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,63,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,127,0.010553599894046783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,127,0.010577599704265594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,255,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,255,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,511,0.012705600261688233
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,511,0.012614400684833526
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,1023,0.028908801078796387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,1023,0.014688000082969666
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,2047,0.047023999691009524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,2047,0.02940959930419922
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,4095,0.08311840295791625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,4095,0.048172798752784726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,1,0.012507200241088867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,1,0.011591999977827071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,3,0.01250080019235611
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,3,0.01130239963531494
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,7,0.01247360035777092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,7,0.011500799655914306
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,15,0.012476799637079239
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,15,0.012628799676895142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,31,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,31,0.012412799894809723
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,63,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,63,0.01249760016798973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,127,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,255,0.012545600533485413
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,255,0.012435200065374375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,511,0.02905920147895813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,511,0.014614400267601014
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,1023,0.047870400547981265
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,1023,0.029414400458335876
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,2047,0.08452320098876953
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,2047,0.049035200476646425
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,4095,0.15970239639282227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,4095,0.08736799955368042
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,1,0.014630399644374847
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,1,0.014627200365066529
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,3,0.014638400077819825
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,3,0.014827199280261993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,7,0.014697599411010741
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,7,0.014523200690746307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,15,0.01464959979057312
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,15,0.014539200067520141
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,31,0.014689600467681885
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,31,0.014617599546909332
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,63,0.016476799547672272
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,63,0.014511999487876893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,127,0.016494399309158324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,127,0.01462240070104599
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,255,0.015467199683189391
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,255,0.014532800018787383
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,511,0.014444799721240997
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,1023,0.012481600046157837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,1023,0.012508800625801087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,127,0.012470400333404541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,2047,0.012347199767827988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,2047,0.02951039969921112
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,2047,0.010552000254392624
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,1,0.011451199650764465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,4095,0.012542399764060973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,1,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,3,0.01239520013332367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,3,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,7,0.012379200011491776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,7,0.011640000343322753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,15,0.012398400157690049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,31,0.012433599680662155
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,31,0.01250240057706833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,63,0.012415999919176102
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,63,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,127,0.012439999729394913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,511,0.012435200065374375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,127,0.012495999783277511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,255,0.012540799379348756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,4095,0.012593600153923034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,255,0.012489599734544754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,511,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,511,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,1023,0.012388800084590913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,1023,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,2047,0.012444800138473511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,2047,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,15,0.01250080019235611
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,4095,0.012620800733566284
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,4095,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,1,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,3,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,3,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,7,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,7,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,15,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,15,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,31,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,31,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,63,0.010545600205659866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,127,0.010548800230026245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,127,0.010547199845314026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,255,0.010571199655532836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,255,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,511,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,511,0.010567999631166457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,1023,0.010505600273609162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,1023,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,2047,0.010593599826097488
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,2047,0.010579200088977813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,4095,0.012564800679683685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,4095,0.01249760016798973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,1,0.010334400087594986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,1,0.010342399775981902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,3,0.010366400331258773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,3,0.009219200164079667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,7,0.010260800272226334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,7,0.009409599751234055
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,1,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,15,0.010316800326108932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,15,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,31,0.010366400331258773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,31,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,63,0.01034879982471466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,127,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,63,0.010500799864530563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,255,0.009361600130796432
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,255,0.008680000156164169
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,511,0.010188800096511842
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,511,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,1023,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,2047,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,2047,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,4095,0.012489599734544754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,4095,0.010499200224876404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,1,0.01659359931945801
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,1,0.01658879965543747
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,3,0.016574400663375854
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,3,0.016704000532627106
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,7,0.016631999611854555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,7,0.0164015993475914
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,15,0.016572800278663636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,15,0.016164800524711607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,31,0.01658719927072525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,31,0.016603200137615202
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,63,0.01660960018634796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,63,0.01658719927072525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,127,0.01653600037097931
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,127,0.01650400012731552
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,63,0.01029599979519844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,255,0.016550399363040924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,511,0.01454080045223236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,127,0.009118399769067764
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,511,0.014526399970054626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,1023,0.012649600207805634
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,1023,0.012470400333404541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,2047,0.012385600060224534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,1023,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,2047,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,4095,0.012467200309038163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,1,0.012503999471664428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,1,0.010751999914646149
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,3,0.012460800260305405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,3,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,7,0.012430399656295776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,7,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,15,0.012404800206422806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,15,0.01242239996790886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,31,0.012406399846076966
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,31,0.012432000041007996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,63,0.012355200201272964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,63,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,127,0.012390399724245072
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,255,0.012342400103807449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,127,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,511,0.012451200187206269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,255,0.010531199723482132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,511,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,1023,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,1023,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,2047,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,2047,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,4095,0.01249760016798973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,4095,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,1,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,1,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,3,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,3,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,7,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,255,0.01528320014476776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,7,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,15,0.010592000186443329
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,15,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,31,0.01053759977221489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,4095,0.012532800436019897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,31,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,127,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,127,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,255,0.01048159971833229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,255,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,511,0.01035040020942688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,511,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,1023,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,1023,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,2047,0.012406399846076966
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,2047,0.01048159971833229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,4095,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,4095,0.0125231996178627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,1,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,1,0.008408000320196151
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,3,0.010288000106811523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,3,0.008441600203514098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,7,0.009627199918031692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,7,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,15,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,15,0.008868800103664398
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,31,0.010494399815797806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,31,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,63,0.009483200311660767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,63,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,63,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,127,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,63,0.010311999917030334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,127,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,255,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,255,0.010345599800348281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,511,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,1023,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,1023,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,2047,0.010503999888896942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,2047,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,4095,0.012596799433231354
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,4095,0.012452799826860428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,1,0.016582399606704712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,1,0.016550399363040924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,3,0.01656160056591034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,3,0.01661120057106018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,7,0.01664000004529953
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,7,0.016523200273513793
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,15,0.016545599699020384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,15,0.01653279960155487
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,31,0.01655520051717758
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,31,0.016551999747753142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,63,0.016527999937534333
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,63,0.016654400527477263
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,127,0.016457599401474
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,255,0.016523200273513793
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,255,0.01650079935789108
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,511,0.01451359987258911
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,511,0.014531199634075165
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,1023,0.01276639997959137
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,1023,0.012558400630950928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,2047,0.014510400593280792
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,2047,0.014473600685596466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,4095,0.01660960018634796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,4095,0.016577599942684172
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,1,0.012491200119256973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,1,0.01234079971909523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,3,0.012455999851226807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,3,0.012404800206422806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,7,0.012486399710178375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,7,0.01249760016798973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,15,0.012468799948692322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,15,0.012412799894809723
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,31,0.012451200187206269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,31,0.012451200187206269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,63,0.012444800138473511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,63,0.012331199645996094
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,127,0.012428800016641617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,127,0.01242400035262108
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,255,0.012467200309038163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,511,0.012417600303888322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,511,0.010496000200510025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,1023,0.012559999525547028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,511,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,1023,0.012467200309038163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,2047,0.014694400131702423
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,2047,0.014440000057220459
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,4095,0.030902400612831116
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,4095,0.01849440038204193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,1,0.009481599926948548
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,1,0.00852160006761551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,127,0.016686399281024934
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,3,0.009550400078296661
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,3,0.008371199667453765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,7,0.009324800223112106
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,7,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,15,0.009723199903964997
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,15,0.008428800106048583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,31,0.00986879989504814
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,31,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,63,0.008489599823951722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,63,0.008390399813652038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,127,0.00841120034456253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,255,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,255,0.008369600027799606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,511,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,511,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,1023,0.012495999783277511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,255,0.012481600046157837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,1023,0.01048159971833229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,2047,0.014548799395561219
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,4095,0.04567199945449829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,4095,0.028889599442481994
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,1,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,1,0.00862239971756935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,3,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,3,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,7,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,7,0.008395200222730636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,15,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,15,0.008715199679136277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,31,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,31,0.008736000210046769
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,63,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,127,0.008560000360012055
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,127,0.010344000160694122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,127,0.00957920029759407
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,255,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,255,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,511,0.01266240030527115
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,511,0.010518400371074677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,1023,0.028988799452781676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,2047,0.028169599175453187
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,1023,0.013396799564361572
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,2047,0.029476800560951234
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,2047,0.04637919962406158
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,4095,0.08335679769515991
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,1,0.014575999975204468
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,4095,0.04794400036334991
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,3,0.01459999978542328
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,3,0.016337600350379945
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,7,0.015145599842071533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,7,0.016499200463294984
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,15,0.014504000544548035
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,15,0.01650879979133606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,31,0.016548800468444824
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,63,0.01658399999141693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,63,0.016527999937534333
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,127,0.0165583997964859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,127,0.016518400609493257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,255,0.016489599645137788
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,255,0.01653600037097931
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,511,0.014412799477577209
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,511,0.012691199779510498
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,63,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,1023,0.012535999715328216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,1023,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,2047,0.0114656001329422
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,4095,0.012534399330615998
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,4095,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,1,0.012452799826860428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,1,0.012452799826860428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,3,0.012455999851226807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,3,0.012515200674533844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,7,0.012435200065374375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,7,0.012465599924325943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,15,0.012454400211572647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,15,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,1,0.016513599455356597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,31,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,31,0.012412799894809723
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,63,0.012465599924325943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,63,0.012414400279521943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,127,0.012428800016641617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,127,0.012495999783277511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,31,0.014526399970054626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,255,0.012449599802494049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,255,0.012936000525951386
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,511,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,511,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,1023,0.010576000064611435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,1023,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,2047,0.01249760016798973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,2047,0.012436799705028534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,4095,0.01449279934167862
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,1,0.01053759977221489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,4095,0.012569600343704223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,1,0.010577599704265594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,3,0.010467199981212616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,3,0.010539200156927109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,7,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,15,0.010467199981212616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,7,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,15,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,31,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,31,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,63,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,2047,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,127,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,127,0.010502400249242783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,255,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,255,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,511,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,511,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,1023,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,1023,0.010553599894046783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,2047,0.012503999471664428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,2047,0.012383999675512314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,4095,0.013299199938774108
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,1,0.008905600011348724
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,1,0.008425600081682205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,3,0.008472000062465668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,3,0.008404800295829773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,7,0.008462399989366532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,7,0.008372800052165985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,15,0.00843840017914772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,15,0.008406399935483932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,31,0.00841279998421669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,31,0.008423999696969987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,63,0.008462399989366532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,63,0.008473599702119828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,127,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,127,0.008353599905967712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,255,0.00840959995985031
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,255,0.008374399691820144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,63,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,511,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,511,0.008388800173997879
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,1023,0.008691199868917466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,2047,0.010553599894046783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,2047,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,4095,0.01260959953069687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,4095,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,1,0.010470400005578995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,4095,0.012627199292182922
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,1,0.010332799702882766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,3,0.01048159971833229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,3,0.010311999917030334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,7,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,7,0.010334400087594986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,15,0.010539200156927109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,15,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,31,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,31,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,63,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,127,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,255,0.010328000038862228
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,255,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,511,0.012460800260305405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,511,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,1023,0.014643199741840363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,1023,0.012654399871826172
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,2047,0.030260801315307617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,2047,0.01669439971446991
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,1,0.012583999335765839
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,1,0.012619200348854064
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,1023,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,3,0.012460800260305405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,7,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,7,0.012455999851226807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,15,0.01252640038728714
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,15,0.012479999661445617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,31,0.012521600723266602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,31,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,63,0.01252799928188324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,63,0.012449599802494049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,127,0.012508800625801087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,127,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,255,0.012878400087356568
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,63,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,255,0.0124719999730587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,127,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,511,0.016616000235080718
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,511,0.014529600739479065
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,1023,0.018614399433135986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,2047,0.048790401220321654
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,2047,0.03260320127010345
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,1,0.014505599439144135
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,1,0.012665599584579468
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,3,0.014524799585342408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,7,0.014537599682807923
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,7,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,15,0.014470399916172027
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,15,0.01255040019750595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,31,0.014519999921321868
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,3,0.012625600397586822
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,31,0.012651200592517852
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,63,0.014552000164985656
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,63,0.012647999823093415
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,127,0.014446400105953217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,127,0.012588800489902496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,255,0.014667199552059173
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,255,0.012606400251388549
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,511,0.029311999678611755
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,511,0.016752000153064727
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,1023,0.049460801482200625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,1023,0.031006398797035217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,2047,0.08618400096893311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,1023,0.030580800771713258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,1,0.0166143998503685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,1,0.014575999975204468
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,3,0.01257600039243698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,3,0.016603200137615202
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,7,0.016553600132465363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,7,0.014545600116252898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,15,0.01658560037612915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,15,0.014508800208568573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,31,0.016622400283813475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,31,0.014567999541759491
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,63,0.016571199893951415
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,63,0.014545600116252898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,127,0.016625599563121797
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,127,0.01451680064201355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,255,0.02884959876537323
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,255,0.014596800506114959
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,511,0.04749760031700134
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,511,0.028977599740028382
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,1023,0.08364959955215454
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,1023,0.048604801297187805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,2047,0.05018079876899719
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,2047,0.15672800540924073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,2047,0.08744800090789795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,1,0.012638400495052337
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,3,0.012720000743865967
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,7,0.01255359947681427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,3,0.012577599287033081
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,7,0.012432000041007996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,15,0.012460800260305405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,15,0.0124208003282547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,31,0.012455999851226807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,31,0.012411200255155564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,63,0.01244639977812767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,63,0.012495999783277511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,127,0.01247360035777092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,127,0.012465599924325943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,255,0.012449599802494049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,255,0.012443199753761292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,511,0.014460800588130951
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,1,0.018667200207710268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,1,0.016697600483894348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,3,0.018803200125694274
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,3,0.016627199947834015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,7,0.018745599687099455
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,7,0.016760000586509706
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,15,0.018723200261592864
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,15,0.01703999936580658
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,31,0.018695999681949616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,31,0.01709119975566864
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,63,0.018718400597572328
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,63,0.017395199835300447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,127,0.018811200559139252
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,127,0.017126399278640746
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,1,0.01244639977812767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,255,0.02048480063676834
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,255,0.017476800084114074
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,511,0.03281440138816834
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,511,0.02237280011177063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,1,0.017051200568675994
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,3,0.020654399693012238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,3,0.016977599263191222
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,7,0.020846399664878845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,7,0.016689600050449373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,15,0.020609599351882935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,15,0.016646400094032288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,511,0.01653439998626709
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,31,0.016774399578571318
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,63,0.02073120027780533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,63,0.017403200268745422
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,127,0.021403199434280394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,127,0.017731200158596038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,255,0.029342401027679443
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,255,0.018456000089645385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,511,0.048039999604225156
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,511,0.02972959876060486
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,1,0.024777600169181825
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,1,0.0227183997631073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,3,0.02481279969215393
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,3,0.022732800245285033
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,7,0.026812800765037538
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,7,0.022731199860572815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,3,0.014681600034236908
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,15,0.026811200380325317
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,15,0.0227743998169899
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,31,0.024876800179481507
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,31,0.023321600258350374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,63,0.026815998554229736
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,63,0.022753599286079406
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,127,0.03145120143890381
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,127,0.022758400440216063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,31,0.020734399557113647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,255,0.030807998776435853
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,511,0.08504639863967896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,511,0.04854240119457245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,1,0.016652800142765045
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,1,0.016577599942684172
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,3,0.01679999977350235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,3,0.01640480011701584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,7,0.016607999801635742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,7,0.014585599303245544
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,15,0.016624000668525696
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,15,0.014604799449443817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,31,0.016899199783802034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,31,0.016731199622154237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,1,0.02082560062408447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,63,0.01681919991970062
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,63,0.01629440039396286
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,127,0.016977599263191222
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,127,0.016553600132465363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,255,0.018739199638366698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,255,0.014705599844455719
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,1,0.02914080023765564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,1,0.026940798759460448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,3,0.02932800054550171
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,7,0.02928000092506409
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,255,0.047812798619270326
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,7,0.027006399631500245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,15,0.028961598873138428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,15,0.027033600211143493
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,31,0.02885279953479767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,31,0.02688480019569397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,63,0.029524800181388856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,63,0.027086400985717775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,127,0.030847999453544616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,127,0.02682879865169525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,255,0.03440000116825104
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,255,0.027191999554634094
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,1,0.026790401339530943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,3,0.03501439988613129
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,3,0.0268095999956131
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,7,0.034985598921775815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,7,0.027286401391029357
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,15,0.035120001435279845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,15,0.026811200380325317
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,31,0.03500480055809021
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,63,0.03505600094795227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,31,0.026769599318504332
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,63,0.026841598749160766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,127,0.03705919981002807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,127,0.026844799518585205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,255,0.04936639964580536
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,255,0.03144159913063049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,1,0.04339359998703003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,1,0.037084800004959104
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,3,0.026888000965118408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,3,0.04328320026397705
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,3,0.037145599722862244
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,7,0.043279999494552614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,15,0.04318400025367737
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,15,0.037169599533081056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,31,0.0371071994304657
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,63,0.04550879895687103
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,63,0.037385600805282596
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,127,0.05346879959106445
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,127,0.04122560024261475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,1,0.03498719930648804
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,255,0.08604000210762024
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,255,0.05250080227851868
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,1,0.01643519997596741
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,1,0.016543999314308167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,3,0.016577599942684172
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,3,0.016571199893951415
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,7,0.016494399309158324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,7,0.016468800604343414
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,15,0.016497600078582763
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,15,0.01619359999895096
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,31,0.016521599888801575
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,31,0.01637279987335205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,63,0.016607999801635742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,63,0.016353599727153778
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,127,0.016505600512027742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,127,0.015007999539375306
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,255,0.016515199840068818
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,255,0.014579200744628906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,511,0.012851199507713318
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,1023,0.01250240057706833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,1023,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,2047,0.012486399710178375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,2047,0.01252799928188324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,4095,0.014552000164985656
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,4095,0.012886400520801543
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,1,0.012425599992275238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,31,0.0452239990234375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,1,0.012470400333404541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,3,0.012505599856376648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,3,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,7,0.012436799705028534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,7,0.012510399520397186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,15,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,15,0.012449599802494049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,31,0.012451200187206269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,31,0.012495999783277511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,63,0.012880000472068786
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,63,0.012494400143623352
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,127,0.012567999958992004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,127,0.012503999471664428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,255,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,255,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,511,0.012510399520397186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,511,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,1023,0.01103999987244606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,511,0.014537599682807923
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,1023,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,2047,0.012604799866676331
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,2047,0.012486399710178375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,4095,0.014523200690746307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,4095,0.014632000029087067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,1,0.010326399654150008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,1,0.010375999659299851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,3,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,3,0.01032480001449585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,7,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,7,0.01035040020942688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,15,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,7,0.03715200126171112
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,31,0.010470400005578995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,31,0.010342399775981902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,63,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,63,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,127,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,255,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,255,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,511,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,511,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,1023,0.010606399923563003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,1023,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,2047,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,2047,0.010550399869680404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,4095,0.012520000338554382
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,4095,0.012433599680662155
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,1,0.010326399654150008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,1,0.008476799726486206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,3,0.010332799702882766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,3,0.010208000242710114
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,7,0.010310400277376175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,7,0.010300800204277039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,15,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,15,0.00854720026254654
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,15,0.01035040020942688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,31,0.008430399745702744
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,63,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,63,0.009745600074529648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,127,0.010340800136327743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,127,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,255,0.010289599746465683
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,255,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,511,0.009099200367927551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,1023,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,1023,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,2047,0.012534399330615998
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,2047,0.010526400059461594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,4095,0.02858400046825409
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,4095,0.013633599877357483
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,1,0.024854399263858795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,1,0.02274399995803833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,3,0.0250463992357254
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,3,0.02276480048894882
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,7,0.02519200146198273
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,7,0.022843199968338012
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,15,0.02507199943065643
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,15,0.022732800245285033
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,31,0.025268799066543578
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,31,0.022673599421977997
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,63,0.02497279942035675
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,63,0.022750400006771088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,127,0.026793599128723145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,31,0.008664000034332275
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,127,0.02269600033760071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,1,0.051446402072906496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,1,0.04648480117321015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,3,0.05145279765129089
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,3,0.04626719951629639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,7,0.05144320130348205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,7,0.04597119987010956
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,15,0.05147839784622192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,15,0.047336000204086306
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,31,0.047203201055526736
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,63,0.054343998432159424
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,63,0.047295999526977536
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,127,0.05753759741783142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,127,0.04835839867591858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,1,0.061724799871444705
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,1,0.04524799883365631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,3,0.06175360083580017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,3,0.04535839855670929
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,7,0.06163679957389832
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,127,0.010251200199127198
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,511,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,7,0.04525760114192963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,15,0.06344000101089478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,15,0.0452320009469986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,31,0.0637391984462738
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,31,0.04525440037250519
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,31,0.05139039754867554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,63,0.06375359892845153
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,63,0.04524799883365631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,127,0.06375679969787598
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,127,0.04726879894733429
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,1,0.08015519976615906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,1,0.06777759790420532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,3,0.08005599975585938
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,3,0.06773920059204101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,7,0.08015679717063903
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,7,0.06782400012016296
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,15,0.08035039901733398
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,15,0.06783040165901184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,31,0.08207359910011292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,31,0.06785119771957397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,63,0.08212320208549499
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,63,0.06958879828453064
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,127,0.09664959907531738
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,127,0.07231680154800416
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,1,0.03705919981002807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,3,0.043243199586868286
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,3,0.03705280125141144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,7,0.04319359958171844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,7,0.037049600481987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,15,0.043161600828170776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,15,0.037084800004959104
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,31,0.04450240135192871
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,31,0.03713119924068451
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,63,0.04539520144462585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,63,0.03714239895343781
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,1,0.09638879895210266
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,1,0.08422080278396607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,3,0.0962544023990631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,3,0.08425599932670594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,7,0.09661920070648193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,7,0.08424800038337707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,15,0.09814720153808594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,15,0.08443040251731873
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,31,0.09939839839935302
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,31,0.08557599782943726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,63,0.10130720138549805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,63,0.08751680254936219
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,1,0.11695040464401245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,1,0.0821727991104126
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,3,0.11697920560836791
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,3,0.08067520260810852
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,7,0.11702560186386109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,7,0.08042880296707153
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,15,0.11694079637527466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,15,0.08210880160331727
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,31,0.11700160503387451
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,31,0.08239200115203857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,63,0.11788320541381836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,1,0.15211199522018432
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,1,0.1271247982978821
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,3,0.15384479761123657
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,3,0.12713279724121093
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,7,0.153713595867157
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,7,0.12721760272979737
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,15,0.15379999876022338
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,15,0.12716799974441528
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,31,0.15377600193023683
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,63,0.1538879990577698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,63,0.1304800033569336
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,1,0.016607999801635742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,1,0.01648160070180893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,3,0.016673600673675536
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,3,0.01653279960155487
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,7,0.016519999504089354
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,7,0.01656319946050644
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,15,0.01652639955282211
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,15,0.016495999693870545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,31,0.01648319959640503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,1,0.0430976003408432
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,63,0.01658080071210861
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,63,0.08248800039291382
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,127,0.01659359931945801
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,127,0.01650719940662384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,255,0.01648640036582947
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,255,0.014727999269962311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,511,0.014505599439144135
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,511,0.012555199861526489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,1023,0.012462399899959564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,1023,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,31,0.12766239643096924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,2047,0.012558400630950928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,2047,0.012495999783277511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,4095,0.014507199823856353
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,4095,0.014523200690746307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,1,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,1,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,3,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,3,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,7,0.012439999729394913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,7,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,15,0.012411200255155564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,15,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,31,0.012428800016641617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,31,0.012467200309038163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,63,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,63,0.012451200187206269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,127,0.01048159971833229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,31,0.01650079935789108
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,255,0.012406399846076966
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,63,0.016633599996566772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,255,0.012374400347471236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,511,0.01058880016207695
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,511,0.010571199655532836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,1023,0.012129600346088409
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,1023,0.010627199709415436
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,2047,0.012511999905109405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,2047,0.0124208003282547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,4095,0.014548799395561219
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,1,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,1,0.010318399965763092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,3,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,3,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,7,0.01053759977221489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,7,0.010311999917030334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,15,0.01056160032749176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,15,0.010545600205659866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,31,0.010555200278759003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,31,0.010476800054311753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,63,0.010539200156927109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,63,0.010523200035095215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,127,0.010567999631166457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,127,0.010507199913263321
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,255,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,255,0.010476800054311753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,511,0.010313600301742554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,511,0.010316800326108932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,127,0.011358399689197541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,1023,0.010604800283908844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,1023,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,2047,0.012542399764060973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,2047,0.012460800260305405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,4095,0.02943519949913025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,4095,0.014511999487876893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,1,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,1,0.008489599823951722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,3,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,7,0.008544000238180161
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,7,0.008507200330495835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,15,0.008427199721336365
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,31,0.008539199829101562
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,31,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,63,0.008502399921417237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,63,0.008508799970149994
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,127,0.008489599823951722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,127,0.00862239971756935
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,255,0.008489599823951722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,255,0.008532799780368805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,511,0.010503999888896942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,511,0.008436799794435502
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,1023,0.011155200004577637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,1023,0.01048320010304451
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,2047,0.027302399277687073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,2047,0.012529599666595458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,4095,0.04543200135231018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,4095,0.02741599977016449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,1,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,3,0.008504000306129456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,1,0.01252799928188324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,3,0.012479999661445617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,4095,0.014497600495815277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,7,0.012479999661445617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,15,0.008379200100898742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,7,0.012383999675512314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,31,0.012521600723266602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,15,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,31,0.012486399710178375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,63,0.012468799948692322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,63,0.012383999675512314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,127,0.012462399899959564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,127,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,255,0.012465599924325943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,255,0.012491200119256973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,511,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,511,0.010564800351858139
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,1023,0.012476799637079239
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,1023,0.01260959953069687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,2047,0.014555199444293976
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,2047,0.01449279934167862
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,4095,0.02999039888381958
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,4095,0.018555200099945067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,8191,0.04810880124568939
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,8191,0.031600001454353335
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,1,0.008481600135564805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,3,0.008795200288295746
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,3,0.008502399921417237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,7,0.008504000306129456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,7,0.008551999926567078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,15,0.008511999994516373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,3,0.012529599666595458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,15,0.012608000636100769
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,15,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,31,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,31,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,63,0.008511999994516373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,63,0.008479999750852585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,127,0.008484800159931184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,127,0.008451200276613235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,255,0.008423999696969987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,511,0.010500799864530563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,511,0.01048159971833229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,1023,0.012503999471664428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,1023,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,1,0.008459199965000153
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,2047,0.028372800350189208
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,2047,0.014505599439144135
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,4095,0.045921599864959715
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,4095,0.028857600688934327
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,8191,0.08171200156211852
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,1,0.010505600273609162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,1,0.008417599648237229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,3,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,3,0.008416000008583068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,7,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,7,0.008510400354862214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,15,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,15,0.008475200086832047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,31,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,31,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,63,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,63,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,255,0.008488000184297562
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,127,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,255,0.010556799918413162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,255,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,511,0.01260959953069687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,511,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,1023,0.028911998867988585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,1023,0.01454399973154068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,2047,0.04713920056819916
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,2047,0.028889599442481994
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,4095,0.08296800255775452
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,4095,0.047233599424362185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,8191,0.04720160067081451
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,1,0.012443199753761292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,8191,0.0838479995727539
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,8191,0.15548800230026244
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,1,0.011747200042009354
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,3,0.012564800679683685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,3,0.012374400347471236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,7,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,7,0.011398400366306304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,15,0.012588800489902496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,31,0.012579199671745301
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,15,0.010478399693965912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,63,0.012624000012874604
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,63,0.010470400005578995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,127,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,127,0.01048159971833229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,255,0.012539200484752655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,255,0.010574399679899215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,511,0.028921601176261903
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,511,0.014553600549697876
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,1023,0.047419199347496034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,1023,0.029531198740005492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,2047,0.08391519784927368
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,2047,0.04812160134315491
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,127,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,4095,0.15844000577926637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,4095,0.08653600215911865
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,1,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,1,0.010531199723482132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,8191,0.30744960308074953
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,8191,0.16399359703063965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,3,0.010494399815797806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,7,0.012432000041007996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,7,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,15,0.012521600723266602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,15,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,31,0.012432000041007996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,31,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,63,0.012494400143623352
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,63,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,127,0.012555199861526489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,127,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,255,0.012411200255155564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,255,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,511,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,511,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,1023,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,1023,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,2047,0.012439999729394913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,2047,0.010499200224876404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,4095,0.012537600100040435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,4095,0.012427199631929398
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,8191,0.01451839953660965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,8191,0.014556799829006196
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,1,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,1,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,3,0.010523200035095215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,31,0.012439999729394913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,3,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,7,0.01053600013256073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,7,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,15,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,15,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,31,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,31,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,63,0.010500799864530563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,63,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,127,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,127,0.010543999820947647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,255,0.010572800040245056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,255,0.010345599800348281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,511,0.010520000010728836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,511,0.010531199723482132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,1023,0.010711999982595444
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,1023,0.010478399693965912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,2047,0.011214400082826615
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,2047,0.010545600205659866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,4095,0.012430399656295776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,8191,0.014369599521160126
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,8191,0.014460800588130951
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,1,0.010289599746465683
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,1,0.008551999926567078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,3,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,3,0.008761599659919739
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,7,0.01029599979519844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,7,0.010203199833631516
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,15,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,15,0.01034879982471466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,31,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,31,0.008529599756002426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,63,0.01029599979519844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,63,0.008537600189447403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,127,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,127,0.008499199897050858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,255,0.010267200320959092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,255,0.009593600034713745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,511,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,511,0.010310400277376175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,1023,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,1023,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,2047,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,2047,0.010329599678516387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,4095,0.01231359988451004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,4095,0.010539200156927109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,8191,0.014508800208568573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,8191,0.012532800436019897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,1,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,1,0.009473600238561631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,3,0.010294400155544281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,3,0.009406399726867676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,7,0.008606400340795517
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,7,0.009454400092363358
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,15,0.008486399799585343
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,15,0.008433599770069123
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,31,0.009201599657535553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,31,0.009670399874448777
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,63,0.00952799990773201
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,63,0.010356800258159637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,127,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,127,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,255,0.008718399703502655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,255,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,511,0.010356800258159637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,511,0.01035040020942688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,1023,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,1023,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,2047,0.010489600151777268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,2047,0.010486400127410889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,4095,0.01250240057706833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,4095,0.012529599666595458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,8191,0.01451520025730133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,1,0.0124719999730587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,1,0.011585599929094314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,3,0.012478400021791458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,4095,0.012415999919176102
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,3,0.010503999888896942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,7,0.01239520013332367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,7,0.010505600273609162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,15,0.012534399330615998
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,15,0.010521599650382995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,31,0.012432000041007996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,31,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,63,0.012540799379348756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,63,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,127,0.01260959953069687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,127,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,255,0.01255040019750595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,255,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,511,0.012345600128173827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,511,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,1023,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,2047,0.012455999851226807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,2047,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,4095,0.012478400021791458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,4095,0.012412799894809723
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,8191,0.014481599628925323
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,8191,0.012503999471664428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,8191,0.014616000652313232
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,1,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,1,0.010539200156927109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,3,0.010497599840164185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,3,0.010494399815797806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,7,0.010558400303125381
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,7,0.010571199655532836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,15,0.010550399869680404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,15,0.010292799770832061
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,31,0.01056319996714592
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,31,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,63,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,63,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,127,0.010635200142860412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,127,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,255,0.010529600083827972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,255,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,511,0.010558400303125381
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,511,0.010603199899196624
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,1023,0.010556799918413162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,1023,0.010623999685049058
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,2047,0.012412799894809723
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,2047,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,4095,0.012556800246238708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,4095,0.012435200065374375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,8191,0.014635199308395385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,3,0.01053759977221489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,8191,0.014579200744628906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,1,0.009222400188446046
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,3,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,3,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,7,0.009544000029563904
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,7,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,15,0.00942080020904541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,1023,0.012392000108957291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,15,0.010307200253009796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,31,0.01034879982471466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,31,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,63,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,63,0.008595199882984161
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,127,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,127,0.008502399921417237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,255,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,255,0.010316800326108932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,511,0.010363200306892395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,511,0.009616000205278396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,1023,0.010313600301742554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,1023,0.010273600369691849
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,2047,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,4095,0.012398400157690049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,4095,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,8191,0.014534400403499603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,8191,0.012452799826860428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,1,0.010302399843931198
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,1,0.009390400350093841
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,3,0.010328000038862228
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,3,0.00835999995470047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,7,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,15,0.010305599868297577
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,15,0.010358399897813796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,31,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,31,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,63,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,63,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,127,0.010259199887514114
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,127,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,255,0.010342399775981902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,255,0.010367999970912933
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,511,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,1,0.00947680026292801
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,511,0.010356800258159637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,1023,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,2047,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,2047,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,4095,0.012518399953842163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,4095,0.012479999661445617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,8191,0.014539200067520141
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,8191,0.012532800436019897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,1,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,2047,0.01098880022764206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,3,0.012438400089740754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,3,0.012455999851226807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,7,0.012401600182056428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,7,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,15,0.012486399710178375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,15,0.01239679977297783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,31,0.012476799637079239
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,31,0.012398400157690049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,63,0.012368000298738479
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,63,0.01242239996790886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,127,0.012374400347471236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,127,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,255,0.01242400035262108
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,255,0.011736000329256058
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,511,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,511,0.010531199723482132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,1023,0.011843200027942657
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,1023,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,1023,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,2047,0.012507200241088867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,2047,0.012435200065374375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,4095,0.014646400511264802
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,4095,0.01438400000333786
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,8191,0.029359999299049377
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,8191,0.017390400171279907
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,1,0.010526400059461594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,1,0.01055999994277954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,1,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,3,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,3,0.010284800082445145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,7,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,7,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,15,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,15,0.010329599678516387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,31,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,31,0.01032159999012947
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,63,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,63,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,7,0.008374399691820144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,127,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,127,0.010553599894046783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,255,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,511,0.01055999994277954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,511,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,1023,0.012388800084590913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,1023,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,2047,0.012481600046157837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,4095,0.029096001386642457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,4095,0.014603200554847717
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,8191,0.04722239971160889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,8191,0.029238399863243104
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,1,0.00838399976491928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,1,0.008342400193214417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,3,0.008395200222730636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,3,0.008371199667453765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,7,0.008366400003433227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,7,0.00841279998421669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,15,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,15,0.008481600135564805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,31,0.008531200140714646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,31,0.008404800295829773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,63,0.008428800106048583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,63,0.008353599905967712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,127,0.008419200032949447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,127,0.008454400300979614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,255,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,255,0.008575999736785888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,255,0.008353599905967712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,511,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,2047,0.01265760064125061
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,511,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,1023,0.010468800365924836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,2047,0.027076798677444457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,2047,0.012470400333404541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,4095,0.04542079865932465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,4095,0.028966400027275085
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,8191,0.08132479786872863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,8191,0.046051201224327085
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,1,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,1,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,3,0.008510400354862214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,7,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,3,0.010582400113344192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,7,0.008484800159931184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,15,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,15,0.008740799874067307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,31,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,63,0.01067039966583252
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,127,0.010592000186443329
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,127,0.010500799864530563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,255,0.010556799918413162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,255,0.009361600130796432
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,511,0.012651200592517852
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,511,0.010639999806880952
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,1023,0.028692799806594848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,1023,0.014590400457382201
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,2047,0.0467056006193161
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,2047,0.02930240035057068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,1023,0.012476799637079239
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,4095,0.08282080292701721
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,4095,0.047635200619697574
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,8191,0.1559008002281189
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,8191,0.08574079871177673
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,1,0.012507200241088867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,1,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,3,0.012664000689983367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,3,0.012460800260305405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,7,0.012534399330615998
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,7,0.012444800138473511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,15,0.012558400630950928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,15,0.0124719999730587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,31,0.0104592002928257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,31,0.012439999729394913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,63,0.012491200119256973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,63,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,63,0.0124208003282547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,127,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,127,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,255,0.012455999851226807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,255,0.011577600240707397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,511,0.01255200058221817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,511,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,1023,0.010543999820947647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,1023,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,2047,0.0124208003282547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,2047,0.010486400127410889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,4095,0.012649600207805634
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,4095,0.0124719999730587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,8191,0.01456640064716339
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,8191,0.014523200690746307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,1,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,3,0.010468800365924836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,3,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,7,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,7,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,15,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,15,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,31,0.0105103999376297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,31,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,63,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,63,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,127,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,127,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,255,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,255,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,511,0.010520000010728836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,31,0.012548799812793731
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,511,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,1023,0.010644800215959548
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,1023,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,2047,0.012417600303888322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,2047,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,4095,0.012747199833393097
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,4095,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,8191,0.014745600521564484
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,8191,0.01454080045223236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,1,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,1,0.00859839990735054
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,3,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,3,0.008523199707269669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,7,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,7,0.008433599770069123
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,15,0.008660800009965896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,31,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,31,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,63,0.009387200325727462
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,63,0.008451200276613235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,127,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,127,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,255,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,255,0.00841120034456253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,511,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,511,0.010318399965763092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,1023,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,1023,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,2047,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,2047,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,4095,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,8191,0.014475199580192565
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,4095,0.012582400441169738
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,8191,0.01255200058221817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,1,0.00841279998421669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,1,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,3,0.008441600203514098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,3,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,7,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,7,0.008353599905967712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,15,0.008380799740552902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,15,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,31,0.008374399691820144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,31,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,63,0.008473599702119828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,63,0.008369600027799606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,127,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,1,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,255,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,15,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,511,0.008451200276613235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,511,0.008500800281763077
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,1023,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,1023,0.008904000371694564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,2047,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,2047,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,4095,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,8191,0.027596798539161683
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,8191,0.013937599956989288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,1,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,1,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,3,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,3,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,7,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,7,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,15,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,15,0.010375999659299851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,31,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,31,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,63,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,63,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,127,0.010468800365924836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,127,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,127,0.010467199981212616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,255,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,255,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,511,0.010596799850463866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,4095,0.010579200088977813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,511,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,1023,0.0124719999730587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,1023,0.014508800208568573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,2047,0.02908160090446472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,4095,0.04726560115814209
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,4095,0.03009760081768036
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,1,0.01064639985561371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,1,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,3,0.010497599840164185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,3,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,7,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,7,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,15,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,15,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,31,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,31,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,63,0.010523200035095215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,63,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,127,0.010569600015878677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,255,0.008574400097131729
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,127,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,255,0.01055999994277954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,255,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,511,0.014431999623775482
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,511,0.012518399953842163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,1023,0.028806400299072266
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,1023,0.014552000164985656
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,2047,0.04725280106067657
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,2047,0.028856000304222106
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,2047,0.01660960018634796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,4095,0.048153600096702574
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,1,0.012467200309038163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,1,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,3,0.012561599910259246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,3,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,7,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,15,0.012468799948692322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,15,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,31,0.012811200320720672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,31,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,63,0.012768000364303589
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,63,0.012478400021791458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,127,0.012452799826860428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,255,0.012539200484752655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,255,0.012270399928092956
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,511,0.02900800108909607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,511,0.01451680064201355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,1023,0.047830399870872495
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,1023,0.028961598873138428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,2047,0.08491680026054382
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,2047,0.04820640087127685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,4095,0.15877439975738525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,4095,0.08487200140953063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,4095,0.08331999778747559
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,1,0.014564800262451171
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,3,0.016758400201797485
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,7,0.012603199481964112
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,3,0.014660799503326416
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,7,0.016708800196647645
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,7,0.014758400619029999
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,15,0.016655999422073364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,15,0.014688000082969666
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,31,0.01666560024023056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,31,0.014535999298095703
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,63,0.01671680063009262
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,63,0.014660799503326416
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,127,0.016542400419712066
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,127,0.014619199931621552
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,255,0.02901279926300049
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,255,0.01467680037021637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,127,0.012902399897575379
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,511,0.04735200107097626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,1023,0.08358719944953918
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,1023,0.048537600040435794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,2047,0.08705599904060364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,4095,0.30142080783843994
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,4095,0.16644959449768065
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,1,0.012505599856376648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,1,0.01257600039243698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,3,0.012619200348854064
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,3,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,1,0.016596800088882445
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,7,0.012620800733566284
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,7,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,15,0.012612800300121307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,15,0.012443199753761292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,31,0.01266240030527115
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,31,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,63,0.012608000636100769
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,63,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,127,0.01260959953069687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,127,0.012564800679683685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,255,0.012507200241088867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,255,0.012564800679683685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,511,0.01661919951438904
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,511,0.014620800316333771
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,1023,0.03102400004863739
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,1023,0.018544000387191773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,1,0.014633600413799287
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,1,0.012542399764060973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,3,0.014591999351978302
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,3,0.01260959953069687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,2047,0.155840003490448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,7,0.014555199444293976
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,7,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,15,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,31,0.014519999921321868
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,31,0.012615999579429627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,63,0.014484800398349762
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,63,0.012454400211572647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,127,0.01451520025730133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,127,0.012468799948692322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,255,0.014574399590492249
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,255,0.012439999729394913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,511,0.028977599740028382
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,511,0.01658399999141693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,1023,0.048209598660469054
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,1023,0.03089439868927002
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,1,0.016568000614643096
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,1,0.014521600306034088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,3,0.016604800522327424
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,3,0.014633600413799287
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,7,0.016542400419712066
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,511,0.02962239980697632
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,7,0.01449120044708252
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,15,0.016659200191497803
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,15,0.014560000598430633
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,31,0.016748799383640288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,31,0.014601600170135499
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,63,0.014584000408649444
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,15,0.014660799503326416
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,127,0.016577599942684172
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,127,0.01465120017528534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,255,0.02900159955024719
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,255,0.01465120017528534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,511,0.04731999933719635
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,511,0.029150399565696716
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,1023,0.08415039777755737
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,1023,0.048502400517463684
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,1,0.024820800125598907
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,1,0.022657600045204163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,3,0.024984000623226164
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,3,0.022710399329662324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,7,0.02481600046157837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,7,0.022566400468349457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,15,0.02481600046157837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,15,0.02263679951429367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,31,0.02483679950237274
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,31,0.022777600586414336
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,63,0.02677280008792877
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,63,0.022627200186252593
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,127,0.030875200033187868
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,127,0.022700800001621245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,255,0.04747999906539917
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,511,0.08405920267105102
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,511,0.04862880110740662
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,1023,0.0865887999534607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,1023,0.1561295986175537
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,1,0.018651199340820313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,1,0.016859200596809388
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,3,0.018670399487018586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,7,0.018648000061511995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,7,0.016964800655841827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,15,0.018632000684738158
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,63,0.016715200245380403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,15,0.016804799437522888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,31,0.018665599822998046
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,31,0.01659200042486191
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,63,0.01865600049495697
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,63,0.01659200042486191
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,127,0.018702399730682374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,127,0.016595199704170227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,255,0.01656319946050644
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,255,0.019167999923229217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,511,0.03188000023365021
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,511,0.02170239984989166
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,1,0.02067520022392273
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,1,0.017828799784183502
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,3,0.02064799964427948
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,3,0.017504000663757326
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,7,0.020694400370121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,7,0.01767839938402176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,15,0.0207056000828743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,15,0.018500800430774688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,31,0.018505600094795228
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,63,0.020948800444602966
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,63,0.01857600063085556
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,127,0.020747199654579163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,127,0.01797119975090027
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,255,0.03102400004863739
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,255,0.017878399789333345
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,511,0.04814079999923706
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,511,0.03112959861755371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,1,0.026707199215888978
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,3,0.016790400445461272
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,1,0.022777600586414336
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,3,0.02675040066242218
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,3,0.022716799378395082
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,7,0.026817598938941957
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,7,0.022841599583625794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,15,0.026787200570106508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,15,0.022734400629997254
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,31,0.026795199513435362
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,31,0.022705599665641785
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,63,0.026918399333953857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,63,0.022728000581264497
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,31,0.02075839936733246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,127,0.030935999751091004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,127,0.02273920029401779
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,255,0.04778720140457153
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,255,0.030483201146125793
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,511,0.04831840097904205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,255,0.02948639988899231
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,1,0.04323680102825165
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,1,0.03709760010242462
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,3,0.04328800141811371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,3,0.03705280125141144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,7,0.037195199728012086
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,15,0.04338079988956452
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,15,0.0370959997177124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,31,0.043635201454162595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,31,0.03711200058460236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,63,0.04525119960308075
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,63,0.03714079856872558
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,127,0.051963198184967044
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,127,0.03995679914951324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,255,0.08559839725494385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,255,0.05233759880065918
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,511,0.15770720243453978
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,511,0.0868943989276886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,1,0.012545600533485413
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,511,0.0850048005580902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,1,0.012535999715328216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,3,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,3,0.0123648002743721
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,7,0.012476799637079239
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,7,0.012385600060224534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,15,0.012460800260305405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,31,0.012371200323104858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,15,0.01234079971909523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,31,0.012476799637079239
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,63,0.012382400035858155
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,127,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,63,0.012479999661445617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,127,0.01242400035262108
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,255,0.01247360035777092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,255,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,511,0.012439999729394913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,511,0.010769599676132202
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,1023,0.010579200088977813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,1023,0.010556799918413162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,2047,0.012503999471664428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,4095,0.014424000680446625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,4095,0.01255040019750595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,8191,0.014691199362277984
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,8191,0.014593599736690522
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,1,0.010567999631166457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,1,0.010548800230026245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,3,0.010513599961996078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,3,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,7,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,7,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,15,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,15,0.010532800108194351
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,31,0.010639999806880952
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,63,0.010545600205659866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,63,0.01048320010304451
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,127,0.010507199913263321
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,127,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,255,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,255,0.010478399693965912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,511,0.01061279997229576
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,511,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,1023,0.010520000010728836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,1023,0.01053600013256073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,2047,0.012332800030708312
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,2047,0.012356799840927125
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,4095,0.01449279934167862
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,7,0.04317759871482849
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,4095,0.012593600153923034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,8191,0.014643199741840363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,8191,0.014424000680446625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,1,0.008473599702119828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,1,0.008497600257396699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,3,0.008451200276613235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,3,0.008550400286912918
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,7,0.00843520015478134
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,7,0.008475200086832047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,2047,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,15,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,31,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,31,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,63,0.008388800173997879
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,63,0.00857120007276535
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,127,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,127,0.00843840017914772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,255,0.008396799862384795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,255,0.00838399976491928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,511,0.008414400368928909
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,511,0.008395200222730636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,1023,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,1023,0.008665599673986436
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,2047,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,2047,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,4095,0.01252799928188324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,4095,0.010502400249242783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,8191,0.028011199831962586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,8191,0.013185599446296692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,1,0.008499199897050858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,1,0.00835999995470047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,3,0.010254400223493576
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,3,0.008390399813652038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,7,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,7,0.008497600257396699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,15,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,15,0.008425600081682205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,31,0.008844800293445587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,31,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,63,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,63,0.008516799658536911
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,127,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,31,0.01055999994277954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,127,0.009518399834632874
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,255,0.010332799702882766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,255,0.008947200328111648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,511,0.010316800326108932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,511,0.008579199761152267
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,1023,0.010577599704265594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,2047,0.012537600100040435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,2047,0.011700800061225891
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,4095,0.02879520058631897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,4095,0.014468799531459808
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,8191,0.045972800254821776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,8191,0.02882080078125
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,1,0.02917119860649109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,1,0.027430400252342224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,3,0.030558401346206666
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,3,0.027385601401329042
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,7,0.030904000997543334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,15,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,7,0.02810559868812561
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,15,0.02699359953403473
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,31,0.03102239966392517
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,31,0.026878398656845093
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,63,0.030955201387405394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,63,0.02705279886722565
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,127,0.03116639852523804
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,255,0.034302398562431335
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,255,0.028921601176261903
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,1,0.03523840010166168
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,1023,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,3,0.03499679863452911
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,3,0.026791998744010927
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,7,0.03499200046062469
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,7,0.02677919864654541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,15,0.034999999403953555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,15,0.026830399036407472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,31,0.034974399209022525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,31,0.026820799708366393
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,63,0.03508319854736328
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,63,0.02680160105228424
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,127,0.037031999230384825
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,127,0.026791998744010927
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,15,0.03055039942264557
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,255,0.04912000000476837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,1,0.04329119920730591
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,255,0.032739201188087465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,1,0.037057599425315856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,3,0.04327360093593598
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,3,0.03704319894313812
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,7,0.04328800141811371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,127,0.02686080038547516
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,7,0.037169599533081056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,15,0.043278399109840396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,15,0.03707520067691803
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,31,0.0452239990234375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,31,0.03712959885597229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,63,0.045256000757217404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,63,0.03830719888210297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,127,0.05274559855461121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,255,0.08601920008659363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,255,0.051815998554229734
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,1,0.02707360088825226
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,1,0.08008319735527039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,1,0.06579359769821166
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,3,0.06661440134048462
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,7,0.07965919971466065
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,7,0.06759200096130372
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,15,0.08022720217704774
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,15,0.06686400175094605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,31,0.08034080266952515
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,31,0.06771039962768555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,63,0.08202080130577087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,63,0.06790879964828492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,127,0.09617599844932556
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,127,0.0705951988697052
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,255,0.16100480556488037
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,255,0.09535999894142151
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,1,0.051545602083206174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,1,0.04736160039901734
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,3,0.05155680179595947
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,3,0.04731679856777191
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,7,0.05143200159072876
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,127,0.04042080044746399
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,7,0.047331199049949646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,15,0.051420801877975465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,15,0.04730400145053863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,31,0.05140640139579773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,31,0.04738079905509949
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,3,0.0801360011100769
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,63,0.05466560125350952
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,63,0.04730879962444305
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,127,0.05739840269088745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,127,0.048044800758361816
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,1,0.04538080096244812
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,3,0.061745601892471316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,3,0.04535999894142151
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,7,0.06165760159492493
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,7,0.04534879922866821
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,15,0.06165120005607605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,15,0.0454255998134613
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,31,0.06370720267295837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,31,0.0453247994184494
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,63,0.06354399919509887
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,63,0.04727199971675873
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,127,0.06363999843597412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,127,0.04770239889621734
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,1,0.08008159995079041
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,1,0.06841279864311219
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,3,0.08015679717063903
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,3,0.06898880004882812
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,7,0.08002240061759949
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,7,0.06924480199813843
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,15,0.08201919794082642
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,15,0.06969119906425476
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,31,0.08161600232124329
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,31,0.0698144018650055
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,63,0.08212000131607056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,63,0.07005760073661804
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,127,0.09800480008125305
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,127,0.07336159944534301
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,1,0.1498304009437561
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,1,0.1252063989639282
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,3,0.15181759595870972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,3,0.12527040243148804
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,1,0.06180959939956665
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,7,0.15185760259628295
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,7,0.12578879594802855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,15,0.12601759433746337
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,15,0.1518944025039673
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,31,0.15170719623565673
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,31,0.1271391987800598
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,63,0.15209920406341554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,63,0.12746720314025878
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,1,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,1,0.01242239996790886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,127,0.13446400165557862
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,3,0.012427199631929398
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,3,0.012404800206422806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,7,0.01255359947681427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,15,0.012436799705028534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,15,0.012427199631929398
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,31,0.012612800300121307
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,63,0.012451200187206269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,63,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,127,0.012628799676895142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,127,0.012491200119256973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,255,0.012627199292182922
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,255,0.012582400441169738
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,511,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,511,0.012379200011491776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,1023,0.01247360035777092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,1023,0.01064319983124733
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,2047,0.012571200728416443
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,2047,0.012436799705028534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,4095,0.014508800208568573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,4095,0.013486400246620178
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,8191,0.016737599670886994
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,8191,0.015695999562740325
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,1,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,1,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,3,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,3,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,127,0.18115359544754028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,7,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,7,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,7,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,15,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,31,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,31,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,63,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,31,0.012452799826860428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,127,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,127,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,255,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,255,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,511,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,511,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,1023,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,1023,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,2047,0.011521600186824799
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,2047,0.010496000200510025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,4095,0.012588800489902496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,4095,0.01250240057706833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,8191,0.02890399992465973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,8191,0.014604799449443817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,1,0.009835200011730194
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,1,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,3,0.010001599788665771
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,3,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,7,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,7,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,15,0.008404800295829773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,15,0.00843520015478134
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,31,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,15,0.01053759977221489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,63,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,63,0.008881600201129913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,63,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,127,0.010355199873447418
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,127,0.008478400111198426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,255,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,255,0.009219200164079667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,511,0.010329599678516387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,511,0.008814399689435959
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,1023,0.010569600015878677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,1023,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,2047,0.012503999471664428
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,2047,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,4095,0.028992000222206115
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,4095,0.013635200262069703
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,8191,0.046593600511550905
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,8191,0.028870400786399842
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,1,0.008511999994516373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,1,0.008390399813652038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,3,0.008535999804735184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,3,0.008324799686670303
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,7,0.008505599945783615
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,7,0.008380799740552902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,15,0.008515200018882752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,15,0.008380799740552902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,31,0.008500800281763077
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,31,0.00852160006761551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,63,0.008489599823951722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,63,0.008380799740552902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,127,0.008470399677753449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,127,0.008372800052165985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,255,0.00843840017914772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,255,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,511,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,511,0.00835840031504631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,1023,0.01244639977812767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,1023,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,2047,0.02723039984703064
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,2047,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,4095,0.02775999903678894
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,8191,0.08137120008468628
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,8191,0.04613440036773682
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,1,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,1,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,3,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,3,0.010521599650382995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,7,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,7,0.010596799850463866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,15,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,15,0.010366400331258773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,31,0.01053759977221489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,63,0.010576000064611435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,63,0.010497599840164185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,127,0.01058719977736473
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,127,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,255,0.010507199913263321
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,255,0.010345599800348281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,511,0.010577599704265594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,511,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,1023,0.012387199699878693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,1023,0.010566399991512298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,2047,0.012649600207805634
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,2047,0.012444800138473511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,4095,0.02900480031967163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,4095,0.014617599546909332
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,8191,0.04736160039901734
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,8191,0.02966879904270172
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,4095,0.04562880098819733
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,16383,0.08293120265007019
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,31,0.008470399677753449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,16383,0.04833920001983642
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,1,0.0084927998483181
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,1,0.008345600217580795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,3,0.00852160006761551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,7,0.008428800106048583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,7,0.008302400261163712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,15,0.008422400057315826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,15,0.008475200086832047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,31,0.008348800241947174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,31,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,63,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,63,0.008321599662303924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,127,0.00846719965338707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,127,0.008350399881601333
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,255,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,255,0.00835840031504631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,511,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,511,0.008462399989366532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,1023,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,1023,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,2047,0.027340799570083618
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,2047,0.0125231996178627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,4095,0.04575360119342804
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,4095,0.02813279926776886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,8191,0.0815455973148346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,8191,0.04586400091648102
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,16383,0.15272159576416017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,3,0.008462399989366532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,1,0.010345599800348281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,16383,0.0829360008239746
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,1,0.010278400033712387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,3,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,3,0.010375999659299851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,31,0.008393599838018417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,7,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,15,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,15,0.01029599979519844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,31,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,31,0.010278400033712387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,63,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,63,0.010335999727249145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,127,0.010038399696350097
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,255,0.010342399775981902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,255,0.009609600156545639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,511,0.012544000148773193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,511,0.01053600013256073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,1023,0.02821280062198639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,1023,0.013240000605583191
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,2047,0.046540799736976626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,2047,0.028832000494003297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,4095,0.08333600163459778
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,4095,0.04731839895248413
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,8191,0.15535520315170287
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,16383,0.2997391939163208
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,7,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,16383,0.15930880308151246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,1,0.012491200119256973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,1,0.01242400035262108
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,3,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,3,0.012326399981975555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,7,0.012436799705028534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,15,0.012451200187206269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,15,0.010878399759531022
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,31,0.012510399520397186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,31,0.011371199786663056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,63,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,63,0.011032000184059143
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,127,0.012563200294971466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,127,0.011753600090742111
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,255,0.012539200484752655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,255,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,511,0.028935998678207397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,511,0.01465280055999756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,1023,0.04718399941921234
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,8191,0.08421120047569275
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,2047,0.08432480096817016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,2047,0.04854399859905243
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,4095,0.15845600366592408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,4095,0.08700640201568603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,7,0.012529599666595458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,127,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,8191,0.3068399906158447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,8191,0.16340160369873047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,1,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,1,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,3,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,16383,0.3197648048400879
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,3,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,7,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,7,0.010582400113344192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,15,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,15,0.010334400087594986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,31,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,31,0.010311999917030334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,63,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,63,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,127,0.010340800136327743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,127,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,255,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,255,0.01034879982471466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,511,0.008984000235795975
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,511,0.010310400277376175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,1023,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,1023,0.010342399775981902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,2047,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,1023,0.029308798909187316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,2047,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,4095,0.012408000230789185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,4095,0.01233920007944107
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,8191,0.014507199823856353
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,8191,0.01412159949541092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,16383,0.016652800142765045
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,16383,0.017744000256061553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,1,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,1,0.010334400087594986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,3,0.009625600278377533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,7,0.010468800365924836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,7,0.010068800300359726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,15,0.010311999917030334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,15,0.009571199864149093
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,31,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,31,0.008532799780368805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,63,0.010356800258159637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,63,0.008531200140714646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,127,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,127,0.010292799770832061
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,16383,0.6046144008636475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,255,0.008388800173997879
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,511,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,511,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,1023,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,1023,0.008548799902200699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,2047,0.010467199981212616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,2047,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,4095,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,4095,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,8191,0.012558400630950928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,3,0.010278400033712387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,8191,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,16383,0.016915200650691985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,16383,0.01656640022993088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,1,0.010335999727249145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,1,0.0083856001496315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,3,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,3,0.008441600203514098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,7,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,15,0.009435199946165086
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,15,0.008380799740552902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,31,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,31,0.008419200032949447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,63,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,63,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,127,0.008470399677753449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,255,0.010332799702882766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,255,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,255,0.010344000160694122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,511,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,511,0.01029760017991066
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,1023,0.01032159999012947
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,1023,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,2047,0.010574399679899215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,2047,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,4095,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,8191,0.014548799395561219
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,8191,0.014284799993038177
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,16383,0.016572800278663636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,16383,0.014905600249767304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,1,0.010329599678516387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,1,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,3,0.010310400277376175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,7,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,7,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,7,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,15,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,15,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,31,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,31,0.010476800054311753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,63,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,63,0.010318399965763092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,127,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,127,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,127,0.008828800171613693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,255,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,511,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,511,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,1023,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,1023,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,2047,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,2047,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,4095,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,4095,0.012478400021791458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,4095,0.0125231996178627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,8191,0.014444799721240997
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,8191,0.012544000148773193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,16383,0.01658399999141693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,1,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,1,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,3,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,3,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,7,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,3,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,7,0.010531199723482132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,15,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,31,0.010545600205659866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,31,0.010304000228643417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,63,0.010494399815797806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,63,0.0106175996363163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,127,0.010580799728631973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,255,0.010289599746465683
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,127,0.010539200156927109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,255,0.010497599840164185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,255,0.010566399991512298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,511,0.010539200156927109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,511,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,1023,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,1023,0.010579200088977813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,2047,0.010496000200510025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,2047,0.010577599704265594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,16383,0.014550399780273438
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,4095,0.012406399846076966
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,4095,0.012435200065374375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,8191,0.014419199526309967
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,8191,0.01257600039243698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,16383,0.018492799997329713
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,16383,0.016601599752902985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,1,0.008428800106048583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,3,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,3,0.008479999750852585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,7,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,7,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,15,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,15,0.008473599702119828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,31,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,31,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,63,0.009233599901199341
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,63,0.008531200140714646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,127,0.008495999872684479
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,127,0.008398400247097015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,255,0.00843520015478134
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,255,0.008462399989366532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,511,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,511,0.008537600189447403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,1023,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,1023,0.008406399935483932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,2047,0.01048320010304451
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,2047,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,4095,0.01053439974784851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,1,0.010275200009346008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,4095,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,8191,0.014535999298095703
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,8191,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,15,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,16383,0.016648000478744505
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,16383,0.014547200500965118
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,1,0.010367999970912933
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,1,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,3,0.009174399822950364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,3,0.008963199704885483
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,7,0.008486399799585343
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,15,0.008692800253629684
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,31,0.010351999849081039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,31,0.008390399813652038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,63,0.01035040020942688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,63,0.008427199721336365
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,127,0.010326399654150008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,127,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,255,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,255,0.00838399976491928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,511,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,511,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,1023,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,2047,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,2047,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,4095,0.012486399710178375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,4095,0.011785600334405899
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,8191,0.014451199769973755
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,8191,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,16383,0.01653600037097931
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,16383,0.014524799585342408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,1,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,1,0.0090768001973629
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,3,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,3,0.008905600011348724
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,7,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,7,0.008574400097131729
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,15,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,15,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,31,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,31,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,63,0.010505600273609162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,7,0.010305599868297577
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,63,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,127,0.010337600111961364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,255,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,255,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,511,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,511,0.008591999858617782
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,1023,0.010500799864530563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,1023,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,1023,0.010358399897813796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,2047,0.010494399815797806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,4095,0.012540799379348756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,4095,0.01247360035777092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,8191,0.014483200013637542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,8191,0.012545600533485413
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,16383,0.03009119927883148
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,16383,0.014521600306034088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,1,0.010367999970912933
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,15,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,127,0.010302399843931198
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,1,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,3,0.010574399679899215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,3,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,7,0.01061440035700798
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,7,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,15,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,15,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,31,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,31,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,63,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,63,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,127,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,2047,0.010550399869680404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,255,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,255,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,511,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,511,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,1023,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,1023,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,2047,0.012270399928092956
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,2047,0.010564800351858139
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,4095,0.012556800246238708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,4095,0.012383999675512314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,8191,0.029246398806571962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,8191,0.014744000136852264
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,16383,0.047363200783729555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,16383,0.02957119941711426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,1,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,1,0.010238400101661682
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,3,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,3,0.010043200105428696
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,7,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,7,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,15,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,15,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,31,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,31,0.008982399851083756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,63,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,63,0.009216000139713288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,127,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,127,0.009491200000047684
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,255,0.010355199873447418
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,255,0.008553600311279297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,511,0.010328000038862228
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,511,0.009268800169229508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,1023,0.010576000064611435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,1023,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,2047,0.012593600153923034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,2047,0.01053600013256073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,4095,0.02892639935016632
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,4095,0.014452800154685974
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,8191,0.04724799990653992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,16383,0.08239039778709412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,127,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,16383,0.04695039987564087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,1,0.00835999995470047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,1,0.00835840031504631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,3,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,3,0.008337599784135818
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,7,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,7,0.008343999832868576
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,15,0.008366400003433227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,15,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,31,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,31,0.008366400003433227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,63,0.008414400368928909
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,63,0.00873280018568039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,127,0.008524800091981888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,127,0.008454400300979614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,255,0.008371199667453765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,511,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,511,0.00931679978966713
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,1023,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,1023,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,2047,0.027515199780464173
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,2047,0.01252799928188324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,4095,0.045433598756790164
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,4095,0.027983999252319335
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,8191,0.08166880011558533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,8191,0.0461872011423111
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,16383,0.15314559936523436
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,1,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,1,0.008462399989366532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,3,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,3,0.009302400052547455
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,7,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,7,0.00979039967060089
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,8191,0.02908639907836914
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,15,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,15,0.010467199981212616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,31,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,31,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,63,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,63,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,127,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,127,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,255,0.010310400277376175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,511,0.012571200728416443
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,255,0.008593600243330002
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,511,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,1023,0.027531200647354127
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,1023,0.013846400380134582
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,2047,0.04577440023422241
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,2047,0.028860801458358766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,4095,0.08229759931564332
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,16383,0.08207200169563293
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,4095,0.04738079905509949
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,8191,0.15527039766311646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,8191,0.08652639985084534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,16383,0.16533119678497316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,16383,0.3000463962554932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,1,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,1,0.010470400005578995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,3,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,3,0.010521599650382995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,7,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,255,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,15,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,15,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,31,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,31,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,63,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,127,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,127,0.01051200032234192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,255,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,255,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,511,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,511,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,1023,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,1023,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,2047,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,2047,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,4095,0.012505599856376648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,4095,0.01249760016798973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,8191,0.014505599439144135
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,8191,0.014575999975204468
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,16383,0.01658080071210861
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,16383,0.016603200137615202
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,1,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,1,0.009446399658918381
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,3,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,3,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,7,0.01029760017991066
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,7,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,7,0.008825600147247314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,15,0.010315199941396713
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,31,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,31,0.008843199908733368
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,63,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,63,0.010318399965763092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,63,0.008638399839401244
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,127,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,255,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,255,0.008425600081682205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,511,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,511,0.009399999678134919
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,1023,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,1023,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,2047,0.010529600083827972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,2047,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,4095,0.012614400684833526
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,4095,0.011832000315189361
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,8191,0.0151296004652977
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,8191,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,16383,0.016568000614643096
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,16383,0.014558400213718414
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,1,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,3,0.009393599629402161
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,3,0.008393599838018417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,7,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,7,0.00875839963555336
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,15,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,15,0.008872000128030777
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,31,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,31,0.010094399750232696
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,63,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,63,0.010132800042629241
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,127,0.009880000352859497
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,127,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,255,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,255,0.00840959995985031
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,511,0.010367999970912933
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,511,0.008462399989366532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,127,0.009494400024414063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,1023,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,1023,0.0104592002928257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,2047,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,2047,0.010592000186443329
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,4095,0.012529599666595458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,4095,0.01239359974861145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,8191,0.014620800316333771
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,8191,0.012537600100040435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,16383,0.031044799089431762
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,1,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,3,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,1,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,7,0.008534400165081025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,15,0.009415999799966813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,7,0.00840959995985031
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,15,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,15,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,31,0.00841279998421669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,31,0.008374399691820144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,63,0.008380799740552902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,63,0.008369600027799606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,127,0.008544000238180161
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,16383,0.01650400012731552
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,127,0.00841279998421669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,255,0.009489600360393525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,255,0.00846880003809929
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,3,0.008352000266313553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,511,0.008390399813652038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,1023,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,1023,0.008523199707269669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,2047,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,2047,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,4095,0.012476799637079239
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,4095,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,8191,0.027480000257492067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,8191,0.013689599931240082
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,16383,0.04576480090618133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,16383,0.02876960039138794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,1,0.008428800106048583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,1,0.008374399691820144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,3,0.008470399677753449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,3,0.008363199979066848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,7,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,7,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,15,0.008459199965000153
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,31,0.008483199775218964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,31,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,63,0.008455999940633774
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,63,0.00835679993033409
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,127,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,127,0.008396799862384795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,255,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,255,0.008363199979066848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,511,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,511,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,1023,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,2047,0.028457599878311157
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,2047,0.014537599682807923
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,511,0.009438399970531464
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,4095,0.046323201060295104
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,4095,0.02889440059661865
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,8191,0.0823535978794098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,8191,0.04719200134277344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,1,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,3,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,3,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,7,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,7,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,15,0.010340800136327743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,15,0.008408000320196151
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,31,0.010367999970912933
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,31,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,63,0.010507199913263321
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,63,0.010316800326108932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,127,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,127,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,255,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,255,0.008473599702119828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,511,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,511,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,1023,0.02821280062198639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,1023,0.01430400013923645
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,1023,0.01051200032234192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,2047,0.029057601094245912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,4095,0.08324480056762695
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,4095,0.04763360023498535
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,1,0.008851200342178345
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,8191,0.15412960052490235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,8191,0.08392959833145142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,1,0.012542399764060973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,1,0.010692799836397171
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,3,0.012375999987125397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,7,0.012539200484752655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,7,0.012366399914026261
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,15,0.008350399881601333
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,15,0.012534399330615998
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,15,0.01154400035738945
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,31,0.012564800679683685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,31,0.011772800236940384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,63,0.01250240057706833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,63,0.011644800007343293
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,127,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,127,0.01228640004992485
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,255,0.012574400007724761
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,255,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,511,0.02892639935016632
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,511,0.014497600495815277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,1023,0.047574400901794434
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,1023,0.02887519896030426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,2047,0.08409759998321534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,2047,0.04747200012207031
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,4095,0.08416799902915954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,2047,0.04680800139904022
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,8191,0.3080480098724365
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,8191,0.15922880172729492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,1,0.016574400663375854
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,1,0.01462240070104599
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,3,0.012702399492263794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,3,0.016704000532627106
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,3,0.014681600034236908
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,7,0.016582399606704712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,7,0.014641599357128143
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,15,0.016616000235080718
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,15,0.014620800316333771
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,31,0.01661919951438904
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,31,0.014561599493026734
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,63,0.01656640022993088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,127,0.01658399999141693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,127,0.014545600116252898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,255,0.027662399411201476
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,255,0.014524799585342408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,511,0.04609279930591583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,511,0.028880000114440918
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,1023,0.08256800174713134
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,4095,0.1585584044456482
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,1023,0.048003199696540835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,2047,0.15479520559310914
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,2047,0.08589599728584289
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,4095,0.16389119625091553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,4095,0.300052809715271
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,8191,0.3194704055786133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,8191,0.5911632061004639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,1,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,3,0.011982399970293045
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,1,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,3,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,63,0.014713600277900696
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,7,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,7,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,15,0.010547199845314026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,15,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,31,0.010539200156927109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,63,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,63,0.010532800108194351
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,127,0.0104592002928257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,127,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,255,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,255,0.010576000064611435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,511,0.012756800651550293
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,511,0.012703999876976013
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,1023,0.028784000873565675
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,1023,0.014688000082969666
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,2047,0.04699040055274963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,2047,0.02964479923248291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,1,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,3,0.012569600343704223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,3,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,7,0.012409599870443344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,15,0.012430399656295776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,15,0.012468799948692322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,31,0.012443199753761292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,31,0.010577599704265594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,63,0.012435200065374375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,63,0.010505600273609162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,127,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,127,0.010574399679899215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,255,0.012438400089740754
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,255,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,511,0.02868959903717041
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,511,0.01448799967765808
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,1023,0.04723199903964996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,1023,0.029518398642539977
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,31,0.011612799763679505
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,2047,0.0842736005783081
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,2047,0.048600000143051145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,1,0.016700799763202667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,1,0.014662399888038635
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,3,0.016539199650287627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,3,0.014636799693107605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,7,0.016513599455356597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,7,0.014603200554847717
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,15,0.016539199650287627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,15,0.01462240070104599
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,31,0.016571199893951415
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,31,0.014696000516414643
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,63,0.016569599509239197
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,63,0.014697599411010741
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,1,0.012534399330615998
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,127,0.014564800262451171
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,7,0.012611199915409089
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,255,0.0146479994058609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,511,0.02911840081214905
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,1023,0.08347520232200623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,1023,0.047737601399421695
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,2047,0.15546560287475586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,2047,0.08425920009613037
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,1,0.02606559991836548
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,1,0.02269120067358017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,3,0.026019200682640076
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,3,0.022708800435066224
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,7,0.025971201062202454
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,7,0.022710399329662324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,15,0.026763200759887695
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,127,0.01659200042486191
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,15,0.022711999714374542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,255,0.028881600499153136
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,31,0.02627519965171814
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,511,0.04721759855747223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,63,0.026796799898147584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,63,0.022835199534893037
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,127,0.031041601300239564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,127,0.022819200158119203
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,255,0.04721759855747223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,255,0.029868799448013305
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,511,0.08252639770507812
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,511,0.04813120067119599
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,1023,0.1548095941543579
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,1023,0.08586720228195191
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,2047,0.16104480028152465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,1,0.014635199308395385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,2047,0.3004832029342651
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,1,0.012971200048923492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,3,0.014535999298095703
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,7,0.014569599926471711
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,7,0.012620800733566284
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,15,0.014507199823856353
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,15,0.012513600289821625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,31,0.014529600739479065
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,31,0.012532800436019897
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,63,0.01451839953660965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,63,0.012479999661445617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,127,0.014535999298095703
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,31,0.022716799378395082
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,127,0.012547199428081513
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,255,0.01454399973154068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,255,0.012587200105190276
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,511,0.029228800535202028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,511,0.01659359931945801
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,1023,0.04769119918346405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,1023,0.030825600028038025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,1,0.016590400040149687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,1,0.01454399973154068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,3,0.01661120057106018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,7,0.016571199893951415
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,3,0.014542399346828461
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,7,0.014528000354766845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,15,0.01451359987258911
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,31,0.01653279960155487
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,31,0.0165583997964859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,63,0.016451199352741242
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,127,0.01722240000963211
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,3,0.013273599743843078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,127,0.016371199488639833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,255,0.028841599822044373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,255,0.015667200088500977
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,511,0.04714399874210358
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,511,0.029488000273704528
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,1023,0.08311840295791625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,1023,0.048492801189422605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,1,0.025553598999977112
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,1,0.022705599665641785
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,3,0.025484800338745117
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,3,0.0227183997631073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,7,0.02534559965133667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,7,0.022676800191402436
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,15,0.025383999943733214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,15,0.02277279943227768
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,31,0.022631999850273133
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,63,0.026723200082778932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,63,0.022716799378395082
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,127,0.03091840147972107
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,127,0.02284640073776245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,255,0.04759199917316437
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,255,0.030564799904823303
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,15,0.016631999611854555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,511,0.08408960103988647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,511,0.048158401250839235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,1023,0.1562880039215088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,63,0.016548800468444824
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,1023,0.08448640108108521
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,1,0.04323520064353943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,1,0.03705439865589142
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,3,0.04321280121803284
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,3,0.037118399143218996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,7,0.04320000112056732
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,7,0.037118399143218996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,15,0.043222400546073916
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,15,0.03703359961509704
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,31,0.04328480064868927
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,31,0.03713920116424561
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,63,0.04526880085468292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,63,0.03716320097446442
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,127,0.05164639949798584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,127,0.04005120098590851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,255,0.08387519717216492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,255,0.05232800245285034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,511,0.1559424042701721
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,511,0.08583999872207641
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,1,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,1,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,1023,0.29933440685272217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,1023,0.16169919967651367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,31,0.025996801257133485
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,3,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,7,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,3,0.010496000200510025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,15,0.01048320010304451
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,15,0.010606399923563003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,31,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,31,0.010569600015878677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,63,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,63,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,127,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,127,0.010468800365924836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,255,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,255,0.010367999970912933
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,511,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,511,0.010292799770832061
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,1023,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,1023,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,2047,0.010558400303125381
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,4095,0.01252480000257492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,4095,0.012539200484752655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,8191,0.014601600170135499
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,8191,0.014478400349617004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,16383,0.016700799763202667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,16383,0.016068799793720244
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,1,0.010366400331258773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,1,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,3,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,3,0.010286399722099304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,7,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,7,0.009030400216579438
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,15,0.010311999917030334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,15,0.009252800047397614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,31,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,31,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,63,0.010316800326108932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,63,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,127,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,127,0.009379199892282485
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,255,0.010278400033712387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,255,0.009115199744701385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,2047,0.012481600046157837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,511,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,511,0.009052799642086029
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,1023,0.010273600369691849
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,1023,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,2047,0.010520000010728836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,2047,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,4095,0.012569600343704223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,4095,0.012281599640846252
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,8191,0.014601600170135499
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,8191,0.012481600046157837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,16383,0.030900800228118898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,16383,0.014721600711345673
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,1,0.009542399644851684
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,1,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,3,0.009480000287294389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,3,0.00838399976491928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,7,0.009598399698734283
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,7,0.008505599945783615
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,15,0.009710399806499482
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,15,0.008488000184297562
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,31,0.010232000052928925
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,31,0.00843999981880188
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,63,0.008531200140714646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,127,0.00838399976491928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,127,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,255,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,255,0.00851840004324913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,511,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,511,0.008475200086832047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,1023,0.009337600320577621
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,1023,0.008724799752235413
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,2047,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,4095,0.012495999783277511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,2047,0.010279999673366546
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,4095,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,8191,0.028860801458358766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,8191,0.014526399970054626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,16383,0.04678719937801361
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,16383,0.029054400324821473
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,1,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,1,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,3,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,3,0.008502399921417237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,7,0.010289599746465683
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,7,0.008696000277996063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,15,0.010305599868297577
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,15,0.008479999750852585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,31,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,31,0.008454400300979614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,63,0.008462399989366532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,127,0.00917920023202896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,127,0.008444800227880477
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,255,0.008671999722719193
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,255,0.00841279998421669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,511,0.009478399902582169
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,511,0.008433599770069123
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,1023,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,1023,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,2047,0.01250080019235611
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,63,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,2047,0.012080000340938568
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,4095,0.028863999247550964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,4095,0.014377599954605103
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,8191,0.04612480103969574
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,7,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,8191,0.02884640097618103
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,16383,0.08230720162391662
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,16383,0.047009599208831784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,1,0.02070239931344986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,1,0.018566399812698364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,3,0.020721599459648132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,3,0.01857440024614334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,7,0.020656000077724456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,7,0.018590399622917177
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,15,0.018515199422836304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,31,0.020852799713611602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,31,0.018544000387191773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,63,0.020817600190639496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,63,0.01857600063085556
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,127,0.02084160000085831
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,127,0.018539200723171233
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,255,0.0296640008687973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,255,0.01857759952545166
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,511,0.048107200860977174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,511,0.031094399094581605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,1,0.026868799328804018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,1,0.02274720072746277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,3,0.026627200841903686
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,3,0.02287999987602234
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,7,0.026774400472640993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,7,0.022832000255584718
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,15,0.026800000667572023
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,15,0.023401600122451783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,31,0.02682720124721527
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,15,0.020750400424003602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,31,0.023177599906921385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,63,0.026782399415969847
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,63,0.022819200158119203
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,63,0.010340800136327743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,127,0.03102239966392517
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,127,0.024515199661254882
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,255,0.030982398986816408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,255,0.04779520034790039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,511,0.08416960239410401
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,1,0.043367999792099
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,1,0.03711999952793121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,3,0.04329279959201813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,3,0.03720960021018982
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,7,0.04320639967918396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,7,0.03718239963054657
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,15,0.04326240122318268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,31,0.0449999988079071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,15,0.0370608001947403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,31,0.03714239895343781
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,63,0.04531840085983276
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,63,0.03701440095901489
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,127,0.052172797918319705
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,127,0.0410288006067276
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,255,0.08536159992218018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,255,0.05134080052375793
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,511,0.15791840553283693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,511,0.08557760119438171
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,1,0.08003680109977722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,3,0.06780319809913635
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,7,0.08006719946861267
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,7,0.06772159934043884
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,15,0.08011680245399475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,15,0.0677183985710144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,31,0.0802511990070343
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,31,0.06777600049972535
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,63,0.08081920146942138
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,63,0.06795359849929809
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,127,0.09440159797668457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,127,0.07090719938278198
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,255,0.15828160047531128
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,255,0.09429439902305603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,511,0.3015471935272217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,1,0.06776319742202759
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,1,0.035113599896430966
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,1,0.028887999057769776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,3,0.0350959986448288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,3,0.02877120077610016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,7,0.03514080047607422
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,7,0.029003199934959412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,15,0.03514080047607422
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,15,0.0287663996219635
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,31,0.03504000008106232
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,31,0.02894560098648071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,63,0.03512639999389648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,63,0.028815999627113342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,511,0.048651200532913205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,127,0.03697119951248169
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,127,0.026897600293159483
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,255,0.04942879974842072
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,1,0.044049599766731264
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,1,0.039027199149131775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,3,0.04343680143356323
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,3,0.03912000060081482
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,7,0.04330880045890808
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,3,0.08004639744758606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,7,0.039103999733924866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,15,0.043275201320648195
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,15,0.03908959925174713
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,31,0.0452208012342453
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,31,0.03917759954929352
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,63,0.04542079865932465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,63,0.03912320137023926
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,127,0.05251039862632752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,127,0.04168159961700439
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,255,0.08580160140991211
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,255,0.054041600227355956
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,255,0.03282879889011383
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,1,0.0798192024230957
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,1,0.0678816020488739
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,3,0.06775680184364319
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,7,0.07994719743728637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,7,0.06778879761695862
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,15,0.08016960024833679
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,15,0.0677839994430542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,511,0.16132960319519044
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,31,0.0805072009563446
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,63,0.08171200156211852
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,63,0.06984639763832093
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,127,0.0944927990436554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,127,0.07200319766998291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,255,0.16066399812698365
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,255,0.09248639941215515
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,1,0.14983680248260497
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,1,0.12520960569381714
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,3,0.14989279508590697
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,3,0.1272271990776062
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,7,0.15177760124206544
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,15,0.15193120241165162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,15,0.12531839609146117
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,31,0.15180319547653198
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,31,0.1272495985031128
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,31,0.06803839802742004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,63,0.15268959999084472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,63,0.12902239561080933
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,127,0.18005119562149047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,127,0.13276159763336182
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,255,0.306878399848938
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,255,0.18111200332641603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,1,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,3,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,1,0.010491199791431427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,3,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,7,0.010601600259542465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,7,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,15,0.010692799836397171
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,15,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,31,0.010625600069761276
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,31,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,63,0.010679999738931656
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,63,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,127,0.010601600259542465
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,127,0.010499200224876404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,255,0.010595200210809707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,511,0.010505600273609162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,7,0.1269327998161316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,511,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,1023,0.010556799918413162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,1023,0.010579200088977813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,2047,0.0124208003282547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,2047,0.011246400326490403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,4095,0.014532800018787383
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,4095,0.012595200538635254
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,8191,0.016275200247764587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,8191,0.014459200203418732
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,16383,0.01671839952468872
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,1,0.008513599634170532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,1,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,3,0.009177599847316743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,3,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,7,0.009009599685668945
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,7,0.008558399975299835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,15,0.009623999893665313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,15,0.008524800091981888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,31,0.009518399834632874
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,31,0.008593600243330002
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,63,0.008505599945783615
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,63,0.008526399731636047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,127,0.008879999816417693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,127,0.008542399853467941
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,255,0.008548799902200699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,255,0.008488000184297562
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,511,0.008524800091981888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,511,0.008531200140714646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,1023,0.010345599800348281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,1023,0.010236799716949463
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,2047,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,2047,0.010355199873447418
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,4095,0.012559999525547028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,4095,0.010660800337791442
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,8191,0.028622400760650635
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,8191,0.014420799911022186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,16383,0.031137600541114807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,16383,0.046851199865341184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,16383,0.028944000601768494
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,1,0.010279999673366546
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,1,0.009678400307893752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,3,0.010358399897813796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,3,0.009726399928331375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,7,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,7,0.010358399897813796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,15,0.010344000160694122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,15,0.008436799794435502
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,31,0.010302399843931198
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,31,0.008761599659919739
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,63,0.010366400331258773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,63,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,127,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,127,0.010375999659299851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,255,0.01029599979519844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,255,0.008539199829101562
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,255,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,511,0.010320000350475311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,511,0.008988799899816513
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,1023,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,2047,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,2047,0.010489600151777268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,4095,0.028569599986076354
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,4095,0.013478399813175201
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,8191,0.04594079852104187
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,8191,0.028854399919509888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,16383,0.08261759877204895
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,16383,0.04642719924449921
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,1,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,1,0.008364800363779068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,3,0.008366400003433227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,3,0.008345600217580795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,7,0.008372800052165985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,7,0.008369600027799606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,15,0.008336000144481659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,15,0.008336000144481659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,31,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,31,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,63,0.008425600081682205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,3,0.07830880284309387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,63,0.008406399935483932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,127,0.008355200290679932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,255,0.008505599945783615
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,255,0.008369600027799606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,511,0.010344000160694122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,511,0.00835840031504631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,1023,0.012436799705028534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,1023,0.010486400127410889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,2047,0.027599999308586122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,2047,0.012430399656295776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,4095,0.04535360038280487
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,8191,0.08183519840240479
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,8191,0.045948800444602964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,16383,0.1535104036331177
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,16383,0.0828383982181549
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,1,0.010337600111961364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,1,0.008441600203514098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,3,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,3,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,7,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,7,0.01032480001449585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,15,0.01035040020942688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,15,0.00889120027422905
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,31,0.010363200306892395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,31,0.008881600201129913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,63,0.010332799702882766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,63,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,127,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,127,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,1023,0.01067200005054474
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,255,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,255,0.008708799630403519
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,511,0.00889120027422905
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,1023,0.010552000254392624
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,4095,0.027428799867630006
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,1023,0.010286399722099304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,2047,0.012505599856376648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,2047,0.010593599826097488
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,4095,0.028460800647735596
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,4095,0.014419199526309967
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,8191,0.04687039852142334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,8191,0.029096001386642457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,16383,0.08209120035171509
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,16383,0.04735040068626404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,32767,0.08320000171661376
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,32767,0.15306559801101685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,1,0.00849440023303032
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,1,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,3,0.008481600135564805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,3,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,7,0.008510400354862214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,7,0.00841120034456253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,15,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,15,0.008374399691820144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,31,0.008393599838018417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,63,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,63,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,127,0.00835840031504631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,127,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,255,0.008374399691820144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,127,0.008396799862384795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,255,0.00838399976491928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,511,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,511,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,1023,0.010649599879980088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,1023,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,2047,0.027599999308586122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,2047,0.012556800246238708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,4095,0.04578079879283905
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,4095,0.02847360074520111
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,8191,0.08181759715080261
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,8191,0.046079999208450316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,16383,0.15222400426864624
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,16383,0.08172159790992736
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,31,0.008510400354862214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,32767,0.29562399387359617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,32767,0.1541792035102844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,1,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,1,0.010313600301742554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,3,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,3,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,7,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,7,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,15,0.01048320010304451
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,15,0.010332799702882766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,31,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,31,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,63,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,511,0.010328000038862228
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,63,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,127,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,127,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,255,0.010335999727249145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,511,0.012615999579429627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,1023,0.028788799047470094
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,1023,0.013238400220870972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,2047,0.04601120054721832
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,2047,0.02850080132484436
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,4095,0.08220959901809692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,4095,0.046803200244903566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,8191,0.15519360303878785
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,8191,0.08392000198364258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,16383,0.30047199726104734
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,16383,0.15791360139846802
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,32767,0.30900959968566893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,32767,0.5925600051879882
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,1,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,255,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,1,0.00838399976491928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,511,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,3,0.00852160006761551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,7,0.00835840031504631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,7,0.008363199979066848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,15,0.008379200100898742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,31,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,31,0.008374399691820144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,63,0.008398400247097015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,63,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,127,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,127,0.008374399691820144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,255,0.008472000062465668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,255,0.008342400193214417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,511,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,511,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,1023,0.008363199979066848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,1023,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,2047,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,2047,0.00973920002579689
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,4095,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,4095,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,8191,0.012520000338554382
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,8191,0.012430399656295776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,16383,0.016579200327396394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,16383,0.01655679941177368
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,32767,0.016550399363040924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,32767,0.01658719927072525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,1,0.01035040020942688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,1,0.008395200222730636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,3,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,3,0.008353599905967712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,7,0.008715199679136277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,7,0.008452799916267396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,15,0.009696000069379807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,15,0.008390399813652038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,31,0.008796799927949905
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,31,0.008433599770069123
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,63,0.00952799990773201
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,63,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,127,0.00867680013179779
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,127,0.00833439975976944
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,255,0.009601599723100662
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,255,0.008406399935483932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,511,0.009438399970531464
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,511,0.008531200140714646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,1023,0.008367999643087386
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,2047,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,2047,0.010886400192975997
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,4095,0.010486400127410889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,3,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,4095,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,8191,0.014440000057220459
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,8191,0.012745599448680877
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,15,0.008771199733018875
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,16383,0.01857919991016388
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,16383,0.01671680063009262
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,32767,0.018801599740982056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,32767,0.018436799943447112
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,1,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,1,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,3,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,3,0.010273600369691849
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,7,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,7,0.010320000350475311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,15,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,31,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,31,0.010320000350475311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,63,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,63,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,127,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,127,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,255,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,255,0.010345599800348281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,511,0.010326399654150008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,511,0.010305599868297577
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,1023,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,1023,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,2047,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,2047,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,4095,0.011472000181674958
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,4095,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,8191,0.014486399292945863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,8191,0.013537600636482239
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,16383,0.01659200042486191
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,16383,0.014528000354766845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,32767,0.019113600254058838
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,32767,0.016595199704170227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,1,0.008473599702119828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,1,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,3,0.008436799794435502
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,15,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,3,0.008454400300979614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,7,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,7,0.008392000198364257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,15,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,15,0.008441600203514098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,31,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,31,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,63,0.010345599800348281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,63,0.008408000320196151
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,127,0.010375999659299851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,127,0.008395200222730636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,255,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,255,0.008708799630403519
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,511,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,511,0.010520000010728836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,1023,0.009440000355243682
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,2047,0.010505600273609162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,2047,0.010304000228643417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,4095,0.010553599894046783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,4095,0.010678400099277497
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,8191,0.012651200592517852
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,8191,0.012590399384498597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,1023,0.010351999849081039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,16383,0.01759359985589981
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,32767,0.018607999384403228
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,32767,0.01857600063085556
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,1,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,1,0.010320000350475311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,3,0.01032159999012947
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,7,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,3,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,7,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,15,0.0102463997900486
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,15,0.008584000170230865
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,31,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,31,0.008462399989366532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,63,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,63,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,127,0.01024480015039444
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,127,0.008575999736785888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,255,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,511,0.010308799892663955
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,511,0.010332799702882766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,1023,0.010470400005578995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,1023,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,2047,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,2047,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,4095,0.012412799894809723
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,4095,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,8191,0.014521600306034088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,8191,0.014508800208568573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,16383,0.016683200001716615
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,1023,0.009415999799966813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,16383,0.016575999557971954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,32767,0.01732639968395233
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,32767,0.016697600483894348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,1,0.00888800024986267
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,1,0.010337600111961364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,3,0.0098191998898983
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,3,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,7,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,7,0.00971520021557808
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,16383,0.016631999611854555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,15,0.010454399883747101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,15,0.00840959995985031
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,31,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,31,0.008419200032949447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,63,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,63,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,127,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,127,0.008483199775218964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,255,0.008980800211429597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,511,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,255,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,1023,0.010313600301742554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,1023,0.010286399722099304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,2047,0.010540799796581268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,2047,0.010556799918413162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,4095,0.012540799379348756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,4095,0.0124208003282547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,8191,0.014433600008487701
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,8191,0.012668800354003907
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,16383,0.016564799845218657
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,16383,0.014521600306034088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,32767,0.03246879875659943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,32767,0.016467200219631196
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,1,0.008588799834251403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,1,0.00843840017914772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,3,0.008540800213813782
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,3,0.008532799780368805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,7,0.008531200140714646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,7,0.008577600121498108
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,15,0.008556800335645676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,255,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,15,0.008422400057315826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,31,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,31,0.00864799991250038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,63,0.008372800052165985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,63,0.008556800335645676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,511,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,127,0.008779200166463852
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,255,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,255,0.00849440023303032
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,511,0.00843999981880188
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,511,0.008481600135564805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,1023,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,1023,0.008694399893283845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,2047,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,2047,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,4095,0.01252480000257492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,4095,0.01242400035262108
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,8191,0.0284527987241745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,8191,0.014655999839305878
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,16383,0.04586719870567322
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,16383,0.029131200909614564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,32767,0.08259999752044678
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,32767,0.0470335990190506
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,1,0.00859680026769638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,1,0.010291200131177902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,3,0.008667200058698653
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,7,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,7,0.008423999696969987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,15,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,15,0.008367999643087386
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,31,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,31,0.00883840024471283
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,63,0.010316800326108932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,63,0.008696000277996063
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,127,0.010278400033712387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,127,0.009391999989748
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,255,0.010257600247859955
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,255,0.008484800159931184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,511,0.010310400277376175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,511,0.009312000125646591
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,1023,0.010489600151777268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,1023,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,2047,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,4095,0.02877120077610016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,4095,0.01443679928779602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,8191,0.04655359983444214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,127,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,8191,0.02913439869880676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,16383,0.08192160129547119
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,16383,0.047249600291252136
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,32767,0.15297600030899047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,1,0.008417599648237229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,1,0.00841120034456253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,3,0.008428800106048583
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,3,0.008529599756002426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,7,0.008363199979066848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,7,0.008347199857234954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,15,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,15,0.008414400368928909
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,31,0.00843520015478134
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,3,0.008590400218963623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,31,0.008367999643087386
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,63,0.008582399785518646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,63,0.00835679993033409
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,127,0.008491200208663941
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,127,0.008444800227880477
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,255,0.008472000062465668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,255,0.008366400003433227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,511,0.010351999849081039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,511,0.008564800024032593
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,1023,0.012415999919176102
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,2047,0.010499200224876404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,1023,0.010571199655532836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,2047,0.02735840082168579
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,2047,0.012567999958992004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,4095,0.04561600089073181
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,4095,0.02876960039138794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,8191,0.08156960010528565
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,8191,0.04637599885463715
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,16383,0.15287359952926635
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,16383,0.08231679797172546
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,32767,0.08300319910049439
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,1,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,32767,0.15440800189971923
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,1,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,32767,0.2967072010040283
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,3,0.010241600126028061
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,7,0.010199999809265137
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,7,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,15,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,15,0.008504000306129456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,31,0.010212799906730652
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,31,0.008687999844551087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,63,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,63,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,127,0.008388800173997879
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,127,0.009556800127029419
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,255,0.008473599702119828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,255,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,511,0.00941760018467903
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,511,0.010320000350475311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,1023,0.00918240025639534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,1023,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,2047,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,2047,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,4095,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,4095,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,8191,0.01446239948272705
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,8191,0.012647999823093415
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,16383,0.016646400094032288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,16383,0.014595200121402741
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,32767,0.01865600049495697
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,32767,0.01650879979133606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,1,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,1,0.008489599823951722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,3,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,3,0.00851999968290329
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,7,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,7,0.009334400296211243
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,15,0.009310399740934372
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,31,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,31,0.010363200306892395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,63,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,63,0.010286399722099304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,127,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,127,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,255,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,255,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,511,0.010275200009346008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,511,0.009352000057697296
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,1023,0.010366400331258773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,1023,0.010318399965763092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,2047,0.010665600001811982
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,4095,0.012511999905109405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,2047,0.0105103999376297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,4095,0.011964800208806992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,8191,0.014486399292945863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,16383,0.016494399309158324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,16383,0.014500799775123595
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,32767,0.03139199912548065
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,32767,0.01656160056591034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,1,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,1,0.009895999729633332
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,3,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,3,0.00843520015478134
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,15,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,7,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,7,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,15,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,31,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,31,0.009124799817800521
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,63,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,63,0.010380800068378448
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,127,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,127,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,255,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,3,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,255,0.009471999853849411
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,511,0.010304000228643417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,511,0.009889599680900574
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,1023,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,1023,0.010363200306892395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,2047,0.010726399719715118
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,2047,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,4095,0.012569600343704223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,4095,0.012443199753761292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,8191,0.014497600495815277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,8191,0.012593600153923034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,16383,0.031016001105308534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,8191,0.012511999905109405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,16383,0.016446399688720702
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,32767,0.0489439994096756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,32767,0.030932798981666565
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,1,0.008515200018882752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,1,0.008505599945783615
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,3,0.008532799780368805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,3,0.008444800227880477
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,7,0.008462399989366532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,7,0.008486399799585343
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,15,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,15,0.008470399677753449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,31,0.008513599634170532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,31,0.008406399935483932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,63,0.008489599823951722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,63,0.008372800052165985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,127,0.008476799726486206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,127,0.00835999995470047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,15,0.010353600233793258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,511,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,511,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,1023,0.010582400113344192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,1023,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,2047,0.027796798944473268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,2047,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,4095,0.046028798818588255
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,4095,0.028067201375961304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,8191,0.08208000063896179
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,8191,0.04668000042438507
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,16383,0.1541792035102844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,16383,0.08233439922332764
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,1,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,1,0.008369600027799606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,3,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,3,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,7,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,255,0.008550400286912918
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,255,0.008366400003433227
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,15,0.010556799918413162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,31,0.010480000078678131
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,15,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,31,0.010305599868297577
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,63,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,63,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,127,0.010623999685049058
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,127,0.010334400087594986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,255,0.010503999888896942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,255,0.010286399722099304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,511,0.012518399953842163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,511,0.010576000064611435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,1023,0.028734400868415833
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,1023,0.012654399871826172
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,2047,0.04649760127067566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,2047,0.02874079942703247
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,4095,0.08256160020828247
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,8191,0.15477440357208253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,8191,0.08396160006523132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,16383,0.2978895902633667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,16383,0.15945279598236084
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,1,0.012542399764060973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,1,0.012408000230789185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,3,0.012336000055074691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,3,0.011737599968910217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,7,0.01249919980764389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,7,0.011648000031709672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,15,0.01173119992017746
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,31,0.012643200159072877
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,31,0.011681599915027619
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,63,0.01252640038728714
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,7,0.008406399935483932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,63,0.012486399710178375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,127,0.01242239996790886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,127,0.012412799894809723
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,255,0.01249760016798973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,255,0.012347199767827988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,511,0.028951999545097352
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,511,0.01467519998550415
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,1023,0.04740000069141388
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,1023,0.029096001386642457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,2047,0.0838703989982605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,4095,0.15899360179901123
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,4095,0.0841584026813507
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,4095,0.04725599884986877
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,8191,0.3069472074508667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,8191,0.15867520570755006
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,16383,0.3077536106109619
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,15,0.012515200674533844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,16383,0.6068319797515869
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,1,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,3,0.008396799862384795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,3,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,7,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,15,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,15,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,31,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,31,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,63,0.010416000336408614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,2047,0.04803839921951294
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,63,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,127,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,127,0.010292799770832061
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,255,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,255,0.010236799716949463
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,511,0.01255359947681427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,511,0.010569600015878677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,1023,0.02866080105304718
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,1023,0.014502400159835815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,2047,0.04635359942913055
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,2047,0.028839999437332155
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,4095,0.08229119777679443
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,1,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,4095,0.04652960002422333
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,1,0.012556800246238708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,1,0.011879999935626984
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,3,0.012518399953842163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,7,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,3,0.011622399836778641
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,7,0.012603199481964112
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,7,0.01218079999089241
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,15,0.01239679977297783
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,31,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,31,0.012347199767827988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,63,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,63,0.012467200309038163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,127,0.01247360035777092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,255,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,255,0.011255999654531479
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,511,0.02875039875507355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,511,0.014579200744628906
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,1023,0.04659039974212646
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,1023,0.029067200422286988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,2047,0.08357440233230591
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,2047,0.047727999091148374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,4095,0.15645120143890381
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,4095,0.08474720120429993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,1,0.01666239947080612
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,1,0.014644800126552582
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,3,0.01656000018119812
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,3,0.014526399970054626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,7,0.016568000614643096
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,15,0.016547200083732606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,15,0.012479999661445617
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,7,0.01475680023431778
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,15,0.01462559998035431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,31,0.016735999286174773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,31,0.014691199362277984
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,63,0.014638400077819825
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,127,0.01671839952468872
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,127,0.012387199699878693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,127,0.014630399644374847
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,255,0.015097600221633912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,511,0.04643679857254028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,511,0.02898080050945282
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,1023,0.08256160020828247
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,1023,0.04711999893188477
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,2047,0.0839792013168335
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,4095,0.3016688108444214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,4095,0.1581104040145874
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,1,0.012627199292182922
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,1,0.012406399846076966
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,3,0.012583999335765839
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,3,0.012464000284671784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,7,0.01249760016798973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,7,0.0124208003282547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,15,0.012598399817943574
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,15,0.0124719999730587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,31,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,31,0.012432000041007996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,63,0.012569600343704223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,63,0.01664479970932007
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,63,0.012350399792194367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,127,0.012495999783277511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,127,0.012430399656295776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,255,0.012406399846076966
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,255,0.012435200065374375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,255,0.028360000252723692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,511,0.014567999541759491
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,1023,0.04729120135307312
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,1023,0.029366400837898255
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,2047,0.0483024001121521
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,2047,0.08398879766464233
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,1,0.016708800196647645
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,1,0.014798399806022645
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,3,0.014734399318695069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,7,0.016633599996566772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,7,0.014659200608730317
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,15,0.016676799952983858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,15,0.014679999649524688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,31,0.016728000342845918
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,31,0.014628799259662628
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,63,0.016731199622154237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,63,0.014692799746990204
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,127,0.01666239947080612
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,127,0.014727999269962311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,255,0.028169599175453187
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,255,0.01459999978542328
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,511,0.046166399121284486
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,511,0.02863999903202057
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,511,0.028862398862838746
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,1023,0.08173440098762512
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,1023,0.048003199696540835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,2047,0.15463520288467408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,3,0.01669120043516159
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,1,0.025519999861717223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,2047,0.15295039415359496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,1,0.02268480062484741
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,3,0.026467201113700867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,3,0.022668799757957457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,7,0.026583999395370483
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,7,0.02271360009908676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,15,0.02476159930229187
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,15,0.02276639938354492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,31,0.02478879988193512
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,31,0.02276960015296936
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,63,0.024753600358963013
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,63,0.022752000391483305
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,127,0.030937600135803222
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,127,0.02282879948616028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,255,0.04615679979324341
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,255,0.030083200335502623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,511,0.0828927993774414
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,511,0.04808480143547058
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,1023,0.0842095971107483
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,1023,0.1547119975090027
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,2047,0.15665119886398315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,2047,0.29954559803009034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,1,0.010486400127410889
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,1,0.008551999926567078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,3,0.00852160006761551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,7,0.010375999659299851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,7,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,15,0.010367999970912933
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,15,0.010360000282526016
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,31,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,2047,0.08480479717254638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,63,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,63,0.010334400087594986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,127,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,127,0.008553600311279297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,255,0.010273600369691849
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,255,0.009163200110197067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,511,0.010340800136327743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,511,0.009444800019264222
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,1023,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,2047,0.010572800040245056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,1023,0.010572800040245056
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,2047,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,4095,0.012567999958992004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,4095,0.012404800206422806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,8191,0.014657600224018097
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,8191,0.01257600039243698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,16383,0.014571200311183929
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,16383,0.01663520038127899
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,32767,0.03116160035133362
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,32767,0.01668799966573715
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,1,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,1,0.008551999926567078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,3,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,7,0.01029599979519844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,3,0.008555199950933456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,3,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,7,0.00929120033979416
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,15,0.00947519987821579
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,15,0.010478399693965912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,31,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,63,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,31,0.008568000048398972
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,63,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,127,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,255,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,127,0.008823999762535095
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,255,0.009540800005197525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,511,0.010307200253009796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,1023,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,511,0.008958400040864945
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,2047,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,2047,0.01061440035700798
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,31,0.01029599979519844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,4095,0.012595200538635254
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,8191,0.012636800110340119
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,16383,0.030859199166297913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,16383,0.01658719927072525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,32767,0.04858080148696899
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,32767,0.03115360140800476
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,1,0.008718399703502655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,1,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,3,0.008515200018882752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,3,0.008454400300979614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,7,0.008446399867534638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,7,0.008377599716186523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,15,0.008422400057315826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,15,0.008433599770069123
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,31,0.008372800052165985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,31,0.008548799902200699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,63,0.008499199897050858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,63,0.008452799916267396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,127,0.008495999872684479
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,1023,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,127,0.008603200316429138
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,255,0.009825599938631057
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,8191,0.014694400131702423
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,255,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,511,0.00857279971241951
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,1023,0.010375999659299851
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,1023,0.008881600201129913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,2047,0.010649599879980088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,2047,0.010320000350475311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,4095,0.0125231996178627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,4095,0.01053600013256073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,8191,0.02879199981689453
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,8191,0.014443199336528777
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,16383,0.046942400932312014
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,16383,0.029135999083518983
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,32767,0.08247200250625611
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,32767,0.047035199403762815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,1,0.01672160029411316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,1,0.016494399309158324
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,3,0.016729600727558136
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,3,0.01648160070180893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,7,0.01669120043516159
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,7,0.01650719940662384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,4095,0.012457600235939026
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,15,0.016710400581359863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,15,0.016495999693870545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,31,0.016651199758052827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,31,0.01653600037097931
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,63,0.016748799383640288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,63,0.01626240015029907
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,127,0.018518400192260743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,127,0.01658399999141693
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,255,0.028828799724578857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,255,0.01576479971408844
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,511,0.04695520102977753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,511,0.029300799965858458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,1023,0.08328480124473572
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,1,0.025224000215530396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,1,0.022724799811840057
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,3,0.025777599215507506
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,3,0.02281759977340698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,7,0.02566399872303009
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,7,0.022812800109386445
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,15,0.026791998744010927
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,15,0.022815999388694764
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,31,0.022865599393844603
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,31,0.026844799518585205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,63,0.026833599805831908
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,63,0.022915199398994446
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,127,0.03091999888420105
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,127,0.022972799837589264
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,255,0.030935999751091004
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,511,0.0828831970691681
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,511,0.04912959933280945
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,511,0.00968960002064705
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,1023,0.1543951988220215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,1,0.04336639940738678
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,1,0.03720319867134094
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,1023,0.04857760071754456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,3,0.04336319863796234
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,3,0.03706879913806915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,7,0.04330880045890808
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,7,0.03710399866104126
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,15,0.043315199017524716
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,31,0.04343520104885101
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,31,0.03729600012302399
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,63,0.045396798849105836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,63,0.03720000088214874
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,127,0.05150399804115295
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,127,0.04049600064754486
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,255,0.08338080048561096
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,255,0.04753440022468567
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,255,0.05003520250320435
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,511,0.15658559799194335
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,511,0.08515999913215637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,1023,0.15770560503005981
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,1023,0.29977281093597413
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,1,0.026766398549079896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,1023,0.08574720025062561
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,1,0.024732799828052522
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,3,0.024724799394607543
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,7,0.026739200949668883
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,7,0.024689599871635437
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,15,0.026707199215888978
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,15,0.024723200500011443
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,31,0.02672959864139557
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,31,0.02470880001783371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,63,0.02683199942111969
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,63,0.02476480007171631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,127,0.024772800505161285
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,127,0.030926400423049928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,15,0.03706560134887695
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,255,0.04728800058364868
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,255,0.03176639974117279
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,511,0.08400800228118896
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,1,0.043279999494552614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,1,0.03910239934921265
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,3,0.04322080016136169
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,3,0.038577601313591
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,7,0.04334560036659241
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,7,0.03899520039558411
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,15,0.04336160123348236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,15,0.039059200882911684
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,31,0.04329279959201813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,31,0.039083200693130496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,63,0.04533919990062714
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,63,0.03913759887218475
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,127,0.05167040228843689
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,127,0.04195359945297241
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,255,0.08455359935760498
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,3,0.02677280008792877
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,255,0.05390239953994751
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,511,0.08701760172843934
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,1,0.08002399802207946
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,1,0.06785439848899841
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,3,0.0802079975605011
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,3,0.0678704023361206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,511,0.049835199117660524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,7,0.08001440167427062
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,7,0.06794880032539367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,15,0.08021280169487
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,15,0.06789439916610718
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,31,0.08012959957122803
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,31,0.06787520051002502
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,63,0.08169280290603638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,127,0.09275519847869873
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,127,0.0727728009223938
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,255,0.15842560529708863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,255,0.09374880194664001
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,511,0.30191841125488283
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,511,0.15590720176696776
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,1,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,511,0.160481595993042
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,1,0.009462399780750275
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,3,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,3,0.008796799927949905
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,7,0.010355199873447418
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,7,0.00928800031542778
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,15,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,31,0.009523200243711472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,31,0.01032159999012947
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,63,0.009489600360393525
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,63,0.009256000071763993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,127,0.01035040020942688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,127,0.010351999849081039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,255,0.01034879982471466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,511,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,511,0.01032480001449585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,63,0.0698207974433899
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,1023,0.010523200035095215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,1023,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,2047,0.010611200332641601
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,4095,0.012641599774360657
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,4095,0.012520000338554382
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,8191,0.014692799746990204
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,8191,0.012571200728416443
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,16383,0.030222401022911072
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,16383,0.01592639982700348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,32767,0.04871360063552856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,32767,0.030953601002693176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,1,0.009249600023031235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,1,0.008483199775218964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,3,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,3,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,7,0.008484800159931184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,7,0.00865119993686676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,15,0.008584000170230865
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,15,0.008564800024032593
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,31,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,15,0.009836799651384353
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,31,0.008472000062465668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,63,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,63,0.00852160006761551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,127,0.008476799726486206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,127,0.008500800281763077
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,255,0.009598399698734283
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,255,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,255,0.008532799780368805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,511,0.008419200032949447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,511,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,1023,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,1023,0.00928959995508194
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,2047,0.010476800054311753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,2047,0.01032480001449585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,4095,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,4095,0.01056160032749176
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,8191,0.014483200013637542
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,2047,0.012265600264072418
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,16383,0.04636319875717163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,16383,0.028790399432182312
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,32767,0.082259202003479
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,32767,0.04669919908046723
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,1,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,3,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,3,0.008540800213813782
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,7,0.008579199761152267
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,7,0.008419200032949447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,15,0.008814399689435959
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,15,0.008591999858617782
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,31,0.009497600048780442
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,31,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,63,0.009297599643468856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,63,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,127,0.008899199962615966
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,127,0.008475200086832047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,255,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,255,0.00846719965338707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,511,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,8191,0.02858720123767853
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,1,0.008416000008583068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,511,0.00857279971241951
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,1023,0.010555200278759003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,1023,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,2047,0.01053600013256073
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,4095,0.02874400019645691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,4095,0.014392000436782838
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,8191,0.04597919881343841
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,8191,0.02900640070438385
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,16383,0.04716959893703461
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,16383,0.08255519866943359
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,32767,0.08158079981803894
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,1,0.00870240032672882
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,1,0.008396799862384795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,3,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,3,0.008579199761152267
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,7,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,7,0.008406399935483932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,15,0.008556800335645676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,15,0.0083856001496315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,31,0.008510400354862214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,31,0.009068799763917923
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,63,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,63,0.009388799965381622
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,127,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,127,0.009721600264310837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,255,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,255,0.008767999708652496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,511,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,1023,0.010452800244092942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,511,0.00958240032196045
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,2047,0.012451200187206269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,2047,0.012564800679683685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,2047,0.010630399733781815
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,4095,0.02874400019645691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,4095,0.014497600495815277
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,8191,0.04633280038833618
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,8191,0.02884320020675659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,16383,0.08367199897766113
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,16383,0.04671840071678161
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,32767,0.15510079860687256
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,32767,0.15421760082244873
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,32767,0.08232799768447877
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,1,0.008371199667453765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,1,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,65535,0.29624640941619873
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,3,0.00835840031504631
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,7,0.008417599648237229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,3,0.008553600311279297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,7,0.008312000334262848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,15,0.008504000306129456
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,15,0.008371199667453765
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,31,0.00856959968805313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,31,0.00833119973540306
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,63,0.008528000116348267
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,63,0.008395200222730636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,127,0.008495999872684479
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,127,0.008345600217580795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,255,0.008347199857234954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,511,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,511,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,1023,0.010694400221109391
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,1023,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,2047,0.02775680124759674
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,2047,0.012521600723266602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,1023,0.010467199981212616
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,4095,0.04585759937763214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,4095,0.027559998631477355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,8191,0.08157439827919007
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,8191,0.0458736002445221
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,16383,0.152838397026062
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,16383,0.08148159980773925
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,32767,0.1530400037765503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,32767,0.2955519914627075
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,65535,0.15552639961242676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,1,0.010478399693965912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,65535,0.30023200511932374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,1,0.009734400361776353
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,65535,0.5784255981445312
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,3,0.01003360003232956
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,7,0.009726399928331375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,255,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,7,0.008479999750852585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,15,0.008590400218963623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,31,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,31,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,63,0.008355200290679932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,63,0.008451200276613235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,127,0.008905600011348724
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,127,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,255,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,255,0.008404800295829773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,511,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,511,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,1023,0.008441600203514098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,1023,0.00836160033941269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,2047,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,2047,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,4095,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,4095,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,8191,0.012476799637079239
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,8191,0.012542399764060973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,16383,0.016516800224781036
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,16383,0.01459999978542328
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,32767,0.016545599699020384
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,32767,0.016548800468444824
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,65535,0.018614399433135986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,65535,0.016689600050449373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,1,0.01050880029797554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,1,0.008478400111198426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,3,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,3,0.008844800293445587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,15,0.010593599826097488
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,7,0.008747199922800064
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,15,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,15,0.008708799630403519
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,31,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,31,0.008675199747085572
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,63,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,63,0.008710400015115739
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,127,0.010539200156927109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,127,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,255,0.0104592002928257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,255,0.010521599650382995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,511,0.010515200346708298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,511,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,1023,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,1023,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,2047,0.010500799864530563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,2047,0.010503999888896942
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,3,0.008459199965000153
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,4095,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,8191,0.012800000607967377
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,8191,0.012462399899959564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,7,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,16383,0.018555200099945067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,32767,0.018385599553585052
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,32767,0.018569600582122803
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,65535,0.02056960016489029
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,65535,0.018772800266742707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,1,0.00836160033941269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,1,0.008963199704885483
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,3,0.008414400368928909
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,7,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,7,0.008502399921417237
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,15,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,15,0.01035040020942688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,31,0.010315199941396713
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,31,0.01024480015039444
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,63,0.010313600301742554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,63,0.009854400157928466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,127,0.010318399965763092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,127,0.010332799702882766
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,255,0.010369600355625152
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,255,0.008513599634170532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,511,0.010345599800348281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,511,0.009198399633169175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,1023,0.010361599922180175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,1023,0.009883199632167817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,2047,0.01035040020942688
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,4095,0.012483199685811996
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,2047,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,4095,0.012488000094890594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,4095,0.01064160019159317
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,8191,0.014510400593280792
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,8191,0.012559999525547028
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,16383,0.018665599822998046
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,16383,0.01669279932975769
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,32767,0.018478399515151976
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,3,0.008497600257396699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,65535,0.018614399433135986
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,1,0.008462399989366532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,1,0.008470399677753449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,3,0.008532799780368805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,3,0.008395200222730636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,7,0.008563199639320373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,7,0.008417599648237229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,15,0.009014400094747544
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,15,0.00851840004324913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,31,0.009198399633169175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,31,0.008408000320196151
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,63,0.0083856001496315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,63,0.008379200100898742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,127,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,127,0.008372800052165985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,255,0.008446399867534638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,255,0.008367999643087386
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,511,0.008454400300979614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,511,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,1023,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,16383,0.01655679941177368
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,1023,0.008425600081682205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,2047,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,2047,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,65535,0.020703999698162077
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,4095,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,4095,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,8191,0.014524799585342408
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,8191,0.014244799315929414
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,16383,0.01656319946050644
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,16383,0.014763200283050537
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,32767,0.018636800348758698
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,32767,0.016673600673675536
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,65535,0.018624000251293182
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,1,0.009583999961614608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,1,0.008499199897050858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,3,0.00950239971280098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,3,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,7,0.010310400277376175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,7,0.008579199761152267
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,15,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,15,0.008463999629020691
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,31,0.008542399853467941
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,31,0.008558399975299835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,63,0.008507200330495835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,63,0.008556800335645676
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,127,0.008683200180530547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,127,0.00854559987783432
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,255,0.008511999994516373
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,32767,0.01870719939470291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,255,0.008580800145864487
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,511,0.009969600290060044
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,511,0.00849440023303032
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,1023,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,1023,0.010340800136327743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,2047,0.010574399679899215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,2047,0.010582400113344192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,4095,0.01260959953069687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,4095,0.01242239996790886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,8191,0.028939199447631837
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,8191,0.014937600493431092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,65535,0.0350847989320755
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,16383,0.046859198808670045
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,32767,0.08260480165481568
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,32767,0.04738560020923614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,65535,0.1536128044128418
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,65535,0.08402559757232667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,1,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,3,0.008436799794435502
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,1,0.010577599704265594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,3,0.008382400125265121
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,7,0.010571199655532836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,7,0.00857279971241951
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,15,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,15,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,31,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,31,0.008657599985599517
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,63,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,63,0.008771199733018875
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,127,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,127,0.0095040000975132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,255,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,255,0.00926079973578453
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,511,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,1023,0.010567999631166457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,1023,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,2047,0.012617599964141846
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,2047,0.010520000010728836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,4095,0.02877599895000458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,4095,0.014508800208568573
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,8191,0.046507200598716734
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,8191,0.0288783997297287
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,16383,0.08228160142898559
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,16383,0.04613600075244904
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,32767,0.15318880081176758
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,32767,0.08229759931564332
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,65535,0.1534432053565979
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,65535,0.2963680028915405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,1,0.008416000008583068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,16383,0.028915199637413024
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,3,0.008380799740552902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,1,0.008455999940633774
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,3,0.009683199971914292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,7,0.009385599941015243
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,15,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,15,0.008617600053548813
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,31,0.010311999917030334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,31,0.008612799644470214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,63,0.010262399911880493
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,63,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,127,0.010304000228643417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,127,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,255,0.010316800326108932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,255,0.008364800363779068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,511,0.010310400277376175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,511,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,1023,0.01027040034532547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,1023,0.008444800227880477
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,2047,0.010487999767065048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,2047,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,4095,0.010627199709415436
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,4095,0.010545600205659866
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,511,0.009297599643468856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,8191,0.01443679928779602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,8191,0.012561599910259246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,16383,0.016550399363040924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,16383,0.014569599926471711
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,32767,0.018622399866580965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,32767,0.01660960018634796
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,65535,0.01860480010509491
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,1,0.009443199634552002
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,1,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,3,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,3,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,7,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,7,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,15,0.00944959968328476
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,15,0.00846719965338707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,31,0.009567999839782714
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,31,0.008497600257396699
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,63,0.009696000069379807
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,7,0.00854559987783432
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,63,0.00846719965338707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,127,0.010473600029945374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,127,0.008580800145864487
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,255,0.010371199995279311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,255,0.00856959968805313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,511,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,511,0.010414399951696397
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,1023,0.011036799848079681
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,1023,0.010425599664449692
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,2047,0.010569600015878677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,2047,0.010497599840164185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,4095,0.012600000202655792
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,4095,0.012462399899959564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,8191,0.014664000272750855
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,8191,0.012495999783277511
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,16383,0.016572800278663636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,16383,0.01443679928779602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,32767,0.03299359977245331
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,32767,0.016569599509239197
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,65535,0.04991520047187805
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,65535,0.03266719877719879
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,1,0.008580800145864487
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,1,0.008379200100898742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,3,0.008566399663686752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,3,0.008369600027799606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,7,0.008524800091981888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,7,0.0083856001496315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,15,0.008417599648237229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,65535,0.034806400537490845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,31,0.008451200276613235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,31,0.008393599838018417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,63,0.008430399745702744
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,127,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,63,0.008303999900817871
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,127,0.00841120034456253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,255,0.008590400218963623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,255,0.008436799794435502
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,511,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,511,0.009219200164079667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,1023,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,1023,0.010505600273609162
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,2047,0.028060799837112425
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,2047,0.012664000689983367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,4095,0.045870399475097655
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,4095,0.028785601258277893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,8191,0.08199520111083984
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,16383,0.1534127950668335
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,32767,0.295849609375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,32767,0.15311520099639891
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,1,0.010577599704265594
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,1,0.009960000216960908
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,3,0.01061440035700798
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,3,0.010113599896430969
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,7,0.01048320010304451
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,7,0.009905599802732468
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,15,0.010585600137710571
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,15,0.008643200248479843
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,31,0.01055999994277954
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,31,0.008526399731636047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,63,0.01051200032234192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,63,0.008561599999666214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,8191,0.04719200134277344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,127,0.008510400354862214
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,255,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,255,0.008454400300979614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,16383,0.08298720121383667
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,511,0.010603199899196624
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,1023,0.027515199780464173
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,1023,0.013833600282669067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,2047,0.04630239903926849
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,2047,0.02893120050430298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,4095,0.08193920254707336
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,4095,0.04716959893703461
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,8191,0.1538432002067566
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,8191,0.08334239721298217
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,16383,0.15807679891586304
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,16383,0.29754559993743895
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,15,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,32767,0.5845471858978272
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,32767,0.3069391965866089
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,1,0.010441599786281586
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,1,0.010118400305509567
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,3,0.010478399693965912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,3,0.010316800326108932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,7,0.010520000010728836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,7,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,15,0.010500799864530563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,15,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,31,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,31,0.008566399663686752
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,63,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,511,0.012513600289821625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,127,0.010225600004196167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,255,0.010463999956846238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,255,0.009964799880981446
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,511,0.012460800260305405
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,511,0.010590399801731109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,1023,0.02757279872894287
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,1023,0.013457599282264709
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,2047,0.045849600434303285
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,2047,0.028913599252700806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,4095,0.08201919794082642
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,8191,0.15493919849395751
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,8191,0.08338239789009094
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,1,0.012513600289821625
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,1,0.012383999675512314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,3,0.012507200241088867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,3,0.010523200035095215
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,7,0.012516799569129943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,7,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,15,0.01252480000257492
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,15,0.010475199669599533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,31,0.012521600723266602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,31,0.010496000200510025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,63,0.012505599856376648
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,63,0.012383999675512314
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,63,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,127,0.012454400211572647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,127,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,255,0.012439999729394913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,255,0.010516799986362457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,511,0.02893120050430298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,511,0.014484800398349762
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,1023,0.04618239998817444
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,1023,0.02885119915008545
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,2047,0.04761280119419098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,2047,0.08363519906997681
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,4095,0.04726879894733429
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,4095,0.08503999710083007
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,4095,0.1567088007926941
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,1,0.012459199875593185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,1,0.012408000230789185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,3,0.012494400143623352
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,8191,0.15867359638214112
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,3,0.012478400021791458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,7,0.012582400441169738
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,15,0.012510399520397186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,7,0.012462399899959564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,31,0.012561599910259246
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,31,0.0124208003282547
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,63,0.012587200105190276
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,63,0.012377600371837615
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,127,0.012639999389648438
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,127,0.012385600060224534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,255,0.012604799866676331
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,255,0.012366399914026261
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,511,0.028721600770950317
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,511,0.014511999487876893
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,1023,0.04701119959354401
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,2047,0.0841264009475708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,2047,0.04749279916286468
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,4095,0.15759520530700682
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,4095,0.08414559960365295
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,1,0.016683200001716615
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,1,0.014640000462532044
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,3,0.016646400094032288
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,3,0.01465120017528534
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,7,0.01666879951953888
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,7,0.014814400672912597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,15,0.01666239947080612
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,15,0.014659200608730317
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,31,0.016631999611854555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,31,0.014571200311183929
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,63,0.016659200191497803
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,63,0.015254400670528412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,127,0.01671680063009262
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,127,0.015588800609111785
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,255,0.02826400101184845
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,255,0.01462559998035431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,511,0.0460783988237381
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,1023,0.02929440140724182
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,511,0.02893120050430298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,1023,0.08176159858703613
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,1023,0.04748319983482361
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,2047,0.08414400219917298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,2047,0.15322719812393187
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,4095,0.29468960762023927
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,4095,0.15743680000305177
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,1,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,1,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,3,0.01034879982471466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,3,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,7,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,7,0.0084927998483181
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,8191,0.3030832052230835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,15,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,31,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,15,0.01244800016283989
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,31,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,63,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,63,0.0103472001850605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,127,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,127,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,255,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,255,0.010328000038862228
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,511,0.00958240032196045
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,511,0.010460799932479859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,1023,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,2047,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,2047,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,4095,0.012625600397586822
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,4095,0.01247360035777092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,8191,0.014505599439144135
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,8191,0.012492799758911132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,16383,0.01656000018119812
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,16383,0.01451680064201355
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,32767,0.031172800064086913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,32767,0.016531200706958772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,65535,0.04973120093345642
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,65535,0.03195199966430664
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,1,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,1,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,3,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,3,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,7,0.010335999727249145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,7,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,15,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,15,0.008380799740552902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,15,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,31,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,31,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,63,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,63,0.010390400141477584
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,127,0.01043360009789467
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,127,0.008721599727869034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,255,0.010329599678516387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,255,0.008743999898433686
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,511,0.01043199971318245
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,511,0.009391999989748
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,1023,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,1023,0.01045759990811348
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,1023,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,2047,0.010582400113344192
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,4095,0.012443199753761292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,8191,0.014588800072669984
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,8191,0.012566399574279786
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,16383,0.030222401022911072
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,16383,0.014529600739479065
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,32767,0.04872480034828186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,32767,0.03094240128993988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,65535,0.08448320031166076
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,65535,0.04914079904556275
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,1,0.016715200245380403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,3,0.01674399971961975
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,3,0.01656640022993088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,7,0.016774399578571318
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,7,0.016524800658226015
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,15,0.016707199811935424
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,15,0.016604800522327424
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,31,0.016705599427223206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,31,0.01656319946050644
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,63,0.01674720048904419
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,63,0.016550399363040924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,127,0.016655999422073364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,127,0.016505600512027742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,255,0.028539198637008666
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,255,0.016468800604343414
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,511,0.04603039920330047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,511,0.029665601253509522
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,1023,0.08259360194206238
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,2047,0.010518400371074677
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,1023,0.04788959920406342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,4095,0.01265760064125061
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,2047,0.1530992031097412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,1,0.026873600482940675
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,2047,0.08374080061912537
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,1,0.02273920029401779
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,3,0.022841599583625794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,7,0.024854399263858795
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,7,0.022859199345111846
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,15,0.02481440007686615
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,15,0.022771200537681578
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,31,0.02489120066165924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,31,0.022815999388694764
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,63,0.024931199848651886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,63,0.02290080040693283
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,127,0.02990880012512207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,127,0.022729599475860597
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,255,0.046188798546791074
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,255,0.031052801012992858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,511,0.08230080008506775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,511,0.04812000095844269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,1023,0.1535104036331177
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,1023,0.08470079898834229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,2047,0.29493439197540283
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,2047,0.15713920593261718
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,1,0.02677600085735321
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,3,0.026848000288009644
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,3,0.02698880136013031
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,1,0.01656640022993088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,3,0.02489600032567978
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,7,0.024900799989700316
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,15,0.026902401447296144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,15,0.024884800612926482
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,31,0.02683840095996857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,31,0.024809600412845613
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,63,0.026899200677871705
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,63,0.024852800369262695
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,127,0.030980798602104186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,127,0.024792000651359558
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,255,0.04725759923458099
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,255,0.03181920051574707
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,511,0.08374879956245422
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,511,0.04959999918937683
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,1,0.02468319982290268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,1023,0.15440959930419923
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,1,0.04317759871482849
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,1023,0.08540480136871338
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,3,0.04329760074615478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,1,0.039113599061965945
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,7,0.02686559855937958
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,3,0.03921439945697784
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,7,0.04314880073070526
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,7,0.03901120126247406
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,15,0.04336639940738678
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,15,0.03932960033416748
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,31,0.039057600498199466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,63,0.045228800177574156
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,63,0.03911679983139038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,127,0.05086399912834168
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,127,0.04119040071964264
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,255,0.08350560069084167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,255,0.05146080255508423
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,511,0.15413600206375122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,511,0.08655999898910523
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,1,0.010329599678516387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,1023,0.2956063985824585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,1,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,1023,0.15877759456634521
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,3,0.010326399654150008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,3,0.010345599800348281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,7,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,15,0.010339199751615524
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,15,0.010311999917030334
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,31,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,31,0.010273600369691849
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,63,0.010385599732398988
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,127,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,127,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,255,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,255,0.010363200306892395
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,511,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,511,0.010284800082445145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,1023,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,1023,0.010254400223493576
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,2047,0.010492800176143647
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,2047,0.010527999699115753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,4095,0.012542399764060973
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,4095,0.012403199821710587
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,8191,0.014575999975204468
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,31,0.0445360004901886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,16383,0.030950400233268737
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,16383,0.016395199298858642
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,32767,0.04883680045604706
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,32767,0.031020799279212953
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,65535,0.08379999995231628
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,65535,0.048187199234962466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,1,0.00838399976491928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,1,0.008390399813652038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,3,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,3,0.008380799740552902
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,7,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,63,0.00894559994339943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,7,0.008416000008583068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,15,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,15,0.008369600027799606
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,31,0.00859680026769638
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,31,0.008416000008583068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,63,0.008553600311279297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,63,0.008393599838018417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,127,0.00852160006761551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,127,0.008379200100898742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,255,0.008507200330495835
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,255,0.008460800349712371
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,511,0.008451200276613235
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,511,0.008433599770069123
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,1023,0.010500799864530563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,1023,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,2047,0.010550399869680404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,2047,0.010489600151777268
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,4095,0.012596799433231354
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,4095,0.010550399869680404
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,8191,0.02884320020675659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,8191,0.012574400007724761
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,8191,0.014476799964904785
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,16383,0.02903040051460266
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,32767,0.0818656027317047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,32767,0.0474592000246048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,7,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,65535,0.1533471941947937
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,1,0.010316800326108932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,65535,0.08281120061874389
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,1,0.010179200023412705
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,3,0.010435199737548828
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,3,0.010310400277376175
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,7,0.010449600219726563
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,7,0.010289599746465683
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,15,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,15,0.010335999727249145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,31,0.010372799634933472
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,31,0.009087999910116195
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,63,0.008905600011348724
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,127,0.010409600287675857
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,127,0.009108799695968627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,255,0.010387200117111205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,255,0.008912000060081481
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,511,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,511,0.008716800063848496
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,1023,0.010524799674749374
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,1023,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,2047,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,16383,0.04709759950637817
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,4095,0.028910401463508605
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,4095,0.014486399292945863
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,8191,0.04688319861888886
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,8191,0.02887200117111206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,16383,0.08316479921340943
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,16383,0.04703199863433838
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,32767,0.1542191982269287
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,32767,0.08266720175743103
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,63,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,65535,0.15565760135650636
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,65535,0.2968672037124634
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,1,0.008455999940633774
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,1,0.008376000076532364
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,131071,0.5841360092163086
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,3,0.008367999643087386
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,7,0.008408000320196151
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,131071,0.2970815896987915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,7,0.008432000130414962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,3,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,15,0.008393599838018417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,15,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,31,0.008363199979066848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,63,0.008379200100898742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,127,0.008823999762535095
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,127,0.008363199979066848
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,255,0.008948799967765809
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,255,0.008336000144481659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,511,0.00838399976491928
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,511,0.008372800052165985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,1023,0.008449599891901017
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,1023,0.008374399691820144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,2047,0.010462400317192078
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,2047,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,4095,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,4095,0.010516799986362457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,8191,0.012529599666595458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,8191,0.012540799379348756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,16383,0.016601599752902985
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,16383,0.015243199467658997
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,32767,0.01759999990463257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,32767,0.016548800468444824
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,65535,0.01871519982814789
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,65535,0.017697599530220032
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,131071,0.023219199478626253
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,131071,0.019108800590038298
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,1,0.010164800286293029
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,2047,0.012591999769210816
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,3,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,3,0.01032480001449585
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,7,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,7,0.01032319962978363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,15,0.010388799756765366
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,15,0.010315199941396713
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,31,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,31,0.010294400155544281
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,63,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,63,0.010331200063228607
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,127,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,127,0.010344000160694122
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,255,0.010451199859380722
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,255,0.009057600051164627
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,511,0.010329599678516387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,511,0.009486400336027146
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,1023,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,31,0.008448000252246856
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,63,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,1,0.008425600081682205
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,2047,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,2047,0.010438399761915207
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,4095,0.0104032002389431
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,8191,0.012563200294971466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,8191,0.012521600723266602
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,16383,0.01674239933490753
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,16383,0.01658560037612915
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,32767,0.018343999981880188
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,32767,0.018622399866580965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,65535,0.0204815998673439
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,65535,0.01863359957933426
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,131071,0.03886240124702454
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,131071,0.020751999318599702
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,1,0.008486399799585343
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,1,0.008454400300979614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,3,0.008422400057315826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,3,0.008459199965000153
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,7,0.008484800159931184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,7,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,15,0.008420799672603608
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,15,0.008441600203514098
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,31,0.008355200290679932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,1023,0.00840959995985031
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,31,0.00836160033941269
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,63,0.008414400368928909
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,63,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,127,0.008393599838018417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,127,0.008401600271463394
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,255,0.00889120027422905
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,511,0.008436799794435502
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,511,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,1023,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,1023,0.008364800363779068
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,2047,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,2047,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,4095,0.01247360035777092
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,4095,0.010558400303125381
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,8191,0.028702399134635924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,8191,0.01467519998550415
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,16383,0.04634079933166504
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,16383,0.02895359992980957
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,32767,0.08213599920272827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,32767,0.04687359929084778
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,65535,0.08343679904937744
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,131071,0.29658079147338867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,131071,0.15483520030975342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,255,0.008403199911117553
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,1,0.010356800258159637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,1,0.008419200032949447
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,3,0.00844319984316826
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,7,0.01040000021457672
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,7,0.009987200051546097
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,15,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,15,0.009921599924564362
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,31,0.010411199927330018
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,31,0.010335999727249145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,63,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,63,0.008452799916267396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,127,0.008595199882984161
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,127,0.008476799726486206
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,255,0.008604799956083297
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,255,0.008534400165081025
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,511,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,511,0.010440000146627427
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,1023,0.01037919968366623
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,65535,0.1537727952003479
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,1023,0.010427200049161912
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,2047,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,4095,0.010635200142860412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,4095,0.010468800365924836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,8191,0.014545600116252898
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,8191,0.01247519999742508
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,16383,0.016568000614643096
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,4095,0.010590399801731109
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,16383,0.014657600224018097
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,32767,0.0165583997964859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,32767,0.01870400011539459
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,3,0.010328000038862228
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,65535,0.035155200958251955
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,65535,0.01870879977941513
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,131071,0.053337597846984865
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,1,0.008472000062465668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,1,0.008388800173997879
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,3,0.008484800159931184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,3,0.0083856001496315
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,7,0.008523199707269669
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,15,0.008535999804735184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,15,0.008454400300979614
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,31,0.008457600325345992
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,31,0.008472000062465668
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,63,0.00852160006761551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,63,0.00852160006761551
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,127,0.008499199897050858
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,127,0.008513599634170532
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,255,0.008444800227880477
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,255,0.008414400368928909
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,511,0.010326399654150008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,2047,0.010446400195360184
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,511,0.008399999886751174
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,1023,0.01165440008044243
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,1023,0.010424000024795533
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,2047,0.02754560112953186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,2047,0.012508800625801087
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,4095,0.04589920043945313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,4095,0.027505600452423097
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,8191,0.08235039710998535
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,131071,0.03563520014286041
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,8191,0.0468639999628067
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,16383,0.15310560464859008
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,7,0.00846560001373291
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,16383,0.0832319974899292
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,32767,0.15307199954986572
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,32767,0.2969840049743652
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,1,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,1,0.008387199789285659
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,65535,0.5833199977874756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,3,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,65535,0.29657440185546874
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,3,0.010329599678516387
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,7,0.01045600026845932
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,15,0.010443200170993806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,31,0.01048159971833229
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,31,0.010356800258159637
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,63,0.01043040007352829
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,63,0.010315199941396713
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,127,0.010401599854230881
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,127,0.010419200360774993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,255,0.010320000350475311
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,255,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,511,0.012531200051307678
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,511,0.010606399923563003
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,1023,0.028364801406860353
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,1023,0.01313599944114685
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,2047,0.04612799882888794
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,2047,0.028590399026870727
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,4095,0.08231840133666993
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,4095,0.04606240093708038
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,8191,0.15515520572662353
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,8191,0.0831167995929718
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,7,0.010436800122261048
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,1,0.012535999715328216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,15,0.010428799688816071
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,16383,0.15481920242309571
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,16383,0.2981920003890991
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,3,0.012518399953842163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,3,0.012388800084590913
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,7,0.012441600114107132
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,15,0.012566399574279786
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,15,0.012081599980592727
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,31,0.012529599666595458
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,31,0.01191840022802353
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,63,0.012593600153923034
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,63,0.011662399768829346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,127,0.01260959953069687
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,127,0.012076800316572189
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,255,0.012577599287033081
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,255,0.011457599699497223
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,511,0.0289247989654541
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,511,0.014572800695896148
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,1023,0.04734239876270294
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,2047,0.08372640013694763
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,2047,0.04745759963989258
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,4095,0.15676959753036498
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,4095,0.0836031973361969
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,8191,0.3036288022994995
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,8191,0.15650080442428588
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,1,0.011446399986743927
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,1,0.010384000092744827
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,1,0.008470399677753449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,3,0.010465600341558457
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,3,0.009547200053930283
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,7,0.010351999849081039
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,7,0.010225600004196167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,15,0.010377600044012069
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,15,0.010396800190210342
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,31,0.01037440001964569
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,31,0.010391999781131745
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,63,0.010382399708032609
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,63,0.010420800000429154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,127,0.010447999835014344
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,127,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,255,0.01085439994931221
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,255,0.010335999727249145
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,511,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,511,0.010417599976062775
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,1023,0.02885279953479767
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,1023,0.010304000228643417
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,1023,0.00963200032711029
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,2047,0.010407999902963639
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,2047,0.010497599840164185
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,4095,0.012507200241088867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,8191,0.014588800072669984
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,8191,0.012545600533485413
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,16383,0.016598400473594666
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,16383,0.014505599439144135
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,32767,0.031718400120735166
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,32767,0.016475200653076172
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,65535,0.050195199251174924
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,65535,0.03296160101890564
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,131071,0.0858735978603363
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,131071,0.04999839961528778
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,1,0.01677280068397522
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,1,0.016505600512027742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,3,0.016711999475955964
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,3,0.016531200706958772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,7,0.016631999611854555
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,7,0.016531200706958772
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,15,0.01666080057621002
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,7,0.012412799894809723
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,15,0.016527999937534333
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,31,0.016715200245380403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,31,0.0165583997964859
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,63,0.01672479957342148
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,63,0.016620799899101257
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,127,0.016715200245380403
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,127,0.016505600512027742
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,255,0.028622400760650635
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,255,0.016969600319862367
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,511,0.0461872011423111
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,511,0.029334399104118346
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,1023,0.08214399814605713
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,1023,0.04789919853210449
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,4095,0.012510399520397186
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,2047,0.1537328004837036
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,2047,0.08341919779777526
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,1,0.0267551988363266
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,1,0.02468640059232712
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,4095,0.15521440505981446
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,3,0.026705598831176756
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,7,0.026843199133872987
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,7,0.024718399345874786
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,15,0.026734399795532226
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,15,0.024697600305080412
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,31,0.026748800277709962
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,31,0.024780799448490144
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,63,0.026824000477790832
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,63,0.02478239983320236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,127,0.031611201167106626
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,127,0.02470560073852539
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,255,0.046312001347541806
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,255,0.031193599104881287
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,511,0.08235520124435425
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,511,0.04929920136928558
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,1023,0.1536080002784729
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,1023,0.0844543993473053
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,2047,0.2948496103286743
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,4095,0.29528639316558836
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,1,0.01026879996061325
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,1,0.010444799810647965
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,3,0.02473759949207306
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,3,0.010364799946546554
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,7,0.01048479974269867
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,7,0.008452799916267396
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,15,0.010406400263309478
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,15,0.008574400097131729
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,31,0.01040479987859726
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,31,0.008459199965000153
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,63,0.010366400331258773
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,63,0.008475200086832047
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,127,0.01034879982471466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,127,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,255,0.010395199805498124
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,255,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,511,0.01034879982471466
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,511,0.010393600165843963
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,1023,0.010471999645233154
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,1023,0.010398399829864503
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,2047,0.010412800312042236
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,4095,0.012486399710178375
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,4095,0.012484800070524216
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,8191,0.014521600306034088
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,8191,0.012518399953842163
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,16383,0.03073279857635498
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,16383,0.014547200500965118
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,32767,0.0482448011636734
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,32767,0.03107680082321167
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,65535,0.08406239748001099
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,65535,0.048702400922775266
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,131071,0.15474400520324708
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,131071,0.08506399989128113
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,3,0.010422399640083313
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,2047,0.012107200175523757
SGLang,0.5.9,NVIDIA GB300,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,2047,0.15544320344924928
