framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,1,0.04700640141963959
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,3,0.04702720046043396
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,7,0.046076801419258115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,1,0.037303999066352844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,15,0.037302398681640626
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,7,0.03831680119037628
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,31,0.04703679978847504
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,31,0.03859840035438537
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,63,0.050220799446105954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,63,0.03905119895935059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,15,0.045372799038887024
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,127,0.05482400059700012
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,127,0.04111039936542511
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,255,0.08371840119361877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,255,0.047276800870895384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,511,0.15189599990844727
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,1023,0.2905551910400391
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,1023,0.15400639772415162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,2047,0.30018880367279055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,2047,0.5671535968780518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,1,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,3,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,3,0.03922879993915558
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,1,0.008868800103664398
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,7,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,7,0.009239999949932099
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,3,0.010369600355625152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,15,0.009251199662685394
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,31,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,31,0.008479999750852585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,63,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,63,0.008508799970149994
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,127,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,127,0.008430399745702744
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,255,0.010596799850463866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,255,0.008529599756002426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,511,0.008382400125265121
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,1023,0.012654399871826172
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,1023,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,2047,0.014587199687957764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,2047,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,1,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,1,0.008422400057315826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,3,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,3,0.008472000062465668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,7,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,7,0.00835679993033409
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,15,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,15,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,15,0.008398400247097015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,31,0.010662399977445603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,511,0.0821071982383728
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,63,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,511,0.011043199896812439
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,127,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,127,0.008510400354862214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,255,0.00851999968290329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,511,0.012449599802494049
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,511,0.009198399633169175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,1023,0.012695999443531036
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,1023,0.011158400028944016
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,2047,0.030934399366378783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,2047,0.012628799676895142
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,1,0.028788799047470094
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,1,0.022865599393844603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,3,0.028758400678634645
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,3,0.02290560007095337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,7,0.028883200883865357
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,7,0.022889600694179536
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,15,0.028830400109291075
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,15,0.02277279943227768
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,31,0.008348800241947174
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,31,0.028815999627113342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,63,0.008369600027799606
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,63,0.022767999768257143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,255,0.010596799850463866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,127,0.034360000491142274
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,127,0.02300959974527359
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,255,0.02884800136089325
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,511,0.08408160209655761
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,511,0.04696640074253082
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,1023,0.15076639652252197
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,1023,0.08263520002365113
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,2047,0.2914191961288452
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,31,0.0227183997631073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,2047,0.15546239614486695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,1,0.012468799948692322
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,3,0.012535999715328216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,1,0.010344000160694122
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,3,0.010409600287675857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,7,0.012513600289821625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,7,0.010385599732398988
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,255,0.047495999932289125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,15,0.010406400263309478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,31,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,31,0.01255040019750595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,63,0.012462399899959564
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,63,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,127,0.012455999851226807
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,127,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,255,0.01239359974861145
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,255,0.00963039994239807
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,511,0.014534400403499603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,511,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,1023,0.0310591995716095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,1023,0.013716800510883332
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,2047,0.04775039851665497
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,2047,0.028841599822044373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,1,0.08243039846420289
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,1,0.06868799924850463
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,3,0.08217120170593262
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,3,0.06787199974060058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,7,0.08230720162391662
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,15,0.012494400143623352
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,7,0.06786879897117615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,15,0.08630560040473938
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,15,0.06784800291061402
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,31,0.08640639781951905
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,31,0.06805440187454223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,63,0.0866815984249115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,63,0.06984000205993653
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,127,0.09260640144348145
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,127,0.07013599872589112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,255,0.15303679704666137
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,255,0.08436959981918335
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,511,0.29051361083984373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,511,0.15285760164260864
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,63,0.028799998760223388
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,1023,0.2962512016296387
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,1023,0.563595199584961
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,1,0.15598880052566527
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,1,0.1292207956314087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,3,0.12943040132522582
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,7,0.15832639932632447
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,7,0.12933919429779053
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,15,0.15905760526657103
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,15,0.13003679513931274
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,31,0.1601263999938965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,31,0.13079999685287474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,63,0.16019359827041627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,63,0.13038079738616942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,127,0.17263200283050537
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,127,0.131985604763031
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,255,0.2930864095687866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,1,0.30126879215240476
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,1,0.25039360523223875
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,3,0.30393760204315184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,3,0.25043039321899413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,7,0.30398080348968504
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,7,0.25080959796905516
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,3,0.15592639446258544
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,15,0.3039344072341919
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,15,0.25097439289093015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,31,0.3066751956939697
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,31,0.2510063886642456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,63,0.30566399097442626
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,63,0.2530719995498657
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,127,0.330729603767395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,1,0.014692799746990204
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,3,0.014617599546909332
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,1,0.01249760016798973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,127,0.252783989906311
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,7,0.014494399726390838
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,3,0.01244639977812767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,15,0.014572800695896148
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,7,0.012612800300121307
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,31,0.014526399970054626
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,255,0.15801440477371215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,31,0.012537600100040435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,63,0.014448000490665436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,63,0.01231039986014366
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,127,0.014545600116252898
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,255,0.014577600359916686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,255,0.012505599856376648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,511,0.031825599074363706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,511,0.014632000029087067
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,1023,0.05059999823570251
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,1023,0.028859201073646545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,2047,0.04756959974765777
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,1,0.5947648048400879
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,1,0.49372158050537107
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,3,0.594977617263794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,15,0.01239679977297783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,3,0.4940847873687744
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,127,0.01173119992017746
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,7,0.6005199909210205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,7,0.49391841888427734
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,15,0.5957231998443604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,2047,0.08640959858894348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,15,0.4980016231536865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,31,0.5959775924682618
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,31,0.49227519035339357
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,63,0.6002639770507813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,63,0.49773440361022947
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,1,0.9804960250854492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,1,1.1838496208190918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,3,1.1851792335510254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,3,0.9877823829650879
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,7,0.9797792434692383
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,7,1.1828479766845703
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,15,0.9864480018615722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,15,1.185431957244873
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,1,0.018680000305175783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,1,0.016604800522327424
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,3,0.018726399540901183
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,3,0.01645440012216568
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,31,1.1830623626708985
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,7,0.0186831995844841
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,15,0.018729600310325622
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,31,0.9847647666931152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,15,0.016411200165748596
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,31,0.01865279972553253
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,31,0.01650719940662384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,63,0.018739199638366698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,63,0.016515199840068818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,127,0.018782399594783783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,127,0.0150736004114151
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,255,0.031035199761390686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,255,0.015140800178050995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,511,0.049116799235343934
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,511,0.028918400406837463
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,1023,0.08397120237350464
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,2047,0.1528272032737732
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,2047,0.08303040266036987
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,1,0.037196800112724304
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,1,0.03116160035133362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,3,0.037238401174545285
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,7,0.014683200418949128
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,3,0.03107520043849945
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,7,0.030958399176597595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,15,0.03720319867134094
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,7,0.03705919981002807
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,15,0.031108799576759338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,31,0.03715679943561554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,31,0.031046399474143983
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,63,0.04014239907264709
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,63,0.031118398904800414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,127,0.03291040062904358
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,1023,0.046859198808670045
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,255,0.0657263994216919
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,255,0.03911679983139038
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,511,0.06424800157546998
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,1023,0.2196432113647461
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,1023,0.11804159879684448
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,2047,0.22584159374237062
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,2047,0.42766079902648924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,1,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,3,0.010449600219726563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,127,0.043243199586868286
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,7,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,511,0.11729919910430908
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,15,0.012436799705028534
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,1,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,3,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,31,0.010387200117111205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,63,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,63,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,127,0.010939200222492219
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,127,0.010329599678516387
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,255,0.011032000184059143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,255,0.010391999781131745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,511,0.012465599924325943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,7,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,511,0.010419200360774993
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,1023,0.012595200538635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,1023,0.010956799983978272
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,2047,0.012863999605178833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,15,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,1,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,1,0.008432000130414962
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,3,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,3,0.008588799834251403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,7,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,7,0.0083856001496315
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,15,0.008484800159931184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,31,0.010344000160694122
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,31,0.008459199965000153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,63,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,63,0.008488000184297562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,127,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,127,0.008528000116348267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,31,0.012582400441169738
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,255,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,255,0.008428800106048583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,511,0.011049599945545196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,511,0.010387200117111205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,1023,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,1023,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,2047,0.020660799741744996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,1,0.022899200022220612
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,1,0.02067999988794327
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,3,0.022852799296379088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,3,0.01884319931268692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,7,0.022840000689029694
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,7,0.018670399487018586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,2047,0.012534399330615998
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,15,0.01897439956665039
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,31,0.02284640073776245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,31,0.018783999979496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,15,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,63,0.022729599475860597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,63,0.018779200315475465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,127,0.026654401421546937
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,127,0.01971839964389801
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,255,0.03909280002117157
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,255,0.02067520022392273
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,511,0.06575679779052734
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,511,0.03752799928188324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,1023,0.11887520551681519
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,1023,0.06477119922637939
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,2047,0.012646399438381195
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,2047,0.22520160675048828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,1,0.010678400099277497
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,2047,0.11851040124893189
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,3,0.01072480008006096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,15,0.02298559993505478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,3,0.00981760025024414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,7,0.011215999722480774
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,7,0.010329599678516387
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,15,0.011249600350856781
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,15,0.009972800314426423
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,31,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,63,0.011308799684047698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,63,0.01043199971318245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,127,0.01119839996099472
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,127,0.01053439974784851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,255,0.01069760024547577
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,255,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,511,0.013104000687599182
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,511,0.010705599933862687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,1023,0.020849600434303284
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,1023,0.013996799290180207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,1,0.008833599835634231
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,2047,0.04339680075645447
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,2047,0.018875199556350707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,1,0.05344799757003784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,1,0.06391519904136658
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,3,0.06382079720497132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,3,0.053814399242401126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,7,0.06377919912338256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,31,0.010841599851846694
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,7,0.053590399026870725
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,15,0.06571679711341857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,31,0.053487998247146604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,15,0.053529602289199826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,63,0.06796960234642029
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,63,0.055471998453140256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,127,0.07332320213317871
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,127,0.05628799796104431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,255,0.11727360486984253
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,255,0.06583520174026489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,511,0.2200160026550293
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,511,0.11752159595489502
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,1,0.11866719722747802
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,1,0.09859520196914673
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,1023,0.4247312068939209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,3,0.11849600076675415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,3,0.09855840206146241
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,7,0.1194000005722046
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,15,0.12116960287094117
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,15,0.09860479831695557
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,31,0.12140320539474488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,31,0.1006943941116333
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,31,0.06794080138206482
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,63,0.12284640073776246
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,63,0.10072799921035766
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,127,0.1321727991104126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,127,0.10290399789810181
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,255,0.222107195854187
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,255,0.12178560495376586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,1,0.19172799587249756
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,3,0.2316960096359253
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,1023,0.2240000009536743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,3,0.19133759737014772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,7,0.2315376043319702
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,7,0.19137760400772094
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,15,0.2311552047729492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,7,0.09866399765014648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,15,0.19278719425201415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,31,0.23209760189056397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,31,0.19078240394592286
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,63,0.23420000076293945
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,63,0.1913696050643921
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,1,0.012540799379348756
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,127,0.25080480575561526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,1,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,127,0.19535520076751708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,3,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,7,0.012595200538635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,7,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,15,0.012579199671745301
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,31,0.012707200646400452
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,31,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,63,0.012627199292182922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,63,0.010662399977445603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,127,0.01255040019750595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,1,0.2277440071105957
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,127,0.010545600205659866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,255,0.01250080019235611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,255,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,511,0.020465600490570068
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,511,0.0124719999730587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,1023,0.03911679983139038
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,1023,0.0186831995844841
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,2047,0.06556159853935242
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,2047,0.037215998768806456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,1,0.4486080169677734
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,3,0.0126351997256279
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,15,0.010595200210809707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,3,0.44907360076904296
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,3,0.3745296001434326
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,7,0.4485311985015869
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,7,0.37399680614471437
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,15,0.4480751991271973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,15,0.37368481159210204
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,31,0.44899678230285645
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,31,0.373145604133606
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,63,0.4506415843963623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,63,0.3737231969833374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,1,0.3730463981628418
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,1,0.7398384094238282
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,1,0.888976001739502
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,3,0.8880847930908203
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,3,0.7393216133117676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,7,0.8862480163574219
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,7,0.7436048030853272
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,15,0.8876591682434082
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,15,0.7375376224517822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,1,0.01658879965543747
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,1,0.014596800506114959
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,3,0.01658560037612915
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,3,0.014497600495815277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,7,0.01656319946050644
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,31,0.7414896011352539
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,31,0.8883456230163574
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,15,0.016543999314308167
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,31,0.014502400159835815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,31,0.016612799465656282
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,63,0.016598400473594666
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,63,0.013255999982357025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,127,0.01656160056591034
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,127,0.014532800018787383
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,255,0.01903360038995743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,255,0.014548799395561219
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,511,0.03975520133972168
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,511,0.01929119974374771
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,1023,0.06776959896087646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,1023,0.03859840035438537
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,2047,0.12319519519805908
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,2047,0.06575520038604736
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,1,0.031188800930976868
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,3,0.031159999966621398
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,7,0.01446239948272705
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,15,0.014657600224018097
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,7,0.03284319937229156
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,15,0.03282560110092163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,1,0.026974400877952574
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,31,0.031224000453948974
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,31,0.026984000205993654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,63,0.034020799398422244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,63,0.027131199836730957
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,127,0.03744319975376129
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,127,0.02698560059070587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,255,0.05562880039215088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,255,0.03487679958343506
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,511,0.09886080026626587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,511,0.05598239898681641
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,7,0.02694559991359711
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,15,0.02686080038547516
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,1023,0.10082399845123291
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,2047,0.3554368019104004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,2047,0.19014079570770265
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,1,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,3,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,3,0.026919999718666078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,7,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,1023,0.18459999561309814
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,15,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,1,0.009391999989748
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,3,0.0086496002972126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,31,0.008587200194597244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,31,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,63,0.009731200337409974
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,127,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,7,0.008561599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,127,0.008601599931716919
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,255,0.010449600219726563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,255,0.00862559974193573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,511,0.008657599985599517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,1023,0.01249760016798973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,1023,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,2047,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,2047,0.010692799836397171
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,1,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,1,0.008399999886751174
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,3,0.010470400005578995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,3,0.00849440023303032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,15,0.008806400001049042
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,7,0.010454399883747101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,7,0.008614400029182434
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,15,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,15,0.008433599770069123
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,31,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,31,0.008595199882984161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,63,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,63,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,63,0.008591999858617782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,127,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,127,0.008539199829101562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,255,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,255,0.008595199882984161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,511,0.010595200210809707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,511,0.009600000083446502
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,511,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,1023,0.012702399492263794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,1023,0.010649599879980088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,2047,0.01641920059919357
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,1,0.020721599459648132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,1,0.016683200001716615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,3,0.020950399339199066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,3,0.016707199811935424
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,7,0.020790399610996248
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,7,0.016758400201797485
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,15,0.020777599513530733
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,15,0.016828800737857818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,31,0.020870399475097657
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,31,0.016809600591659545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,63,0.020942400395870208
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,127,0.021566399931907655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,127,0.018615999817848207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,255,0.03392800092697144
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,255,0.018676799535751343
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,511,0.05569599866867066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,511,0.033134400844573975
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,1023,0.09867839813232422
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,1023,0.05575839877128601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,2047,0.18492319583892822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,2047,0.012620800733566284
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,2047,0.10068479776382447
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,1,0.010734400153160096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,1,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,3,0.010628800094127654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,3,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,7,0.010567999631166457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,7,0.010406400263309478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,15,0.010564800351858139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,31,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,63,0.01675039976835251
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,31,0.010384000092744827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,63,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,127,0.010636799782514573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,127,0.008451200276613235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,255,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,255,0.008535999804735184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,511,0.012742400169372559
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,1023,0.01677280068397522
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,1023,0.012595200538635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,2047,0.041494399309158325
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,2047,0.018705600500106813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,1,0.05558879971504212
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,1,0.04537599980831146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,3,0.05558879971504212
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,3,0.045465600490570066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,7,0.05553280115127564
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,7,0.04754559993743897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,15,0.05557119846343994
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,15,0.047391998767852786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,31,0.0576304018497467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,15,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,31,0.047305598855018616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,63,0.05970720052719116
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,63,0.04739679992198944
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,127,0.06346560120582581
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,63,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,127,0.04795039892196655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,255,0.10036799907684327
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,255,0.05892639756202698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,511,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,511,0.18545279502868653
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,511,0.09995840191841125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,1,0.10073120594024658
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,1,0.08427680134773255
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,3,0.10068320035934449
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,1023,0.35625760555267333
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,3,0.08430079817771911
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,7,0.10072480440139771
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,15,0.10331519842147827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,15,0.08443040251731873
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,31,0.10484479665756226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,31,0.08624479770660401
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,63,0.10518399477005005
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,63,0.0864192008972168
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,127,0.1127951979637146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,127,0.08684960007667542
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,255,0.18743519783020018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,255,0.10609920024871826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,1,0.1913424015045166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,1,0.16136640310287476
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,3,0.1931056022644043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,3,0.1608847975730896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,7,0.19475359916687013
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,7,0.1602671980857849
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,15,0.19420479536056517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,1023,0.189793598651886
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,15,0.16108319759368897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,31,0.16030559539794922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,7,0.08441759943962097
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,63,0.19687039852142335
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,63,0.1603119969367981
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,127,0.16269919872283936
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,127,0.21090879440307617
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,1,0.012649600207805634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,1,0.010675200074911118
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,3,0.012611199915409089
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,3,0.010694400221109391
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,7,0.012732799351215362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,7,0.010702399909496308
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,15,0.012583999335765839
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,15,0.010711999982595444
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,31,0.012577599287033081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,31,0.010751999914646149
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,63,0.012639999389648438
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,63,0.01064319983124733
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,127,0.012676799297332763
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,127,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,255,0.010632000118494033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,511,0.016708800196647645
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,511,0.012721599638462066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,1023,0.037248000502586365
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,31,0.19396480321884155
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,1023,0.01690240055322647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,2047,0.06188480257987976
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,2047,0.0366351991891861
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,1,0.3740639925003052
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,1,0.3127968072891235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,3,0.3751807928085327
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,3,0.31296000480651853
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,7,0.3117536067962646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,15,0.3764031887054443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,7,0.37847199440002444
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,15,0.31162400245666505
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,31,0.3771183967590332
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,31,0.3143104076385498
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,255,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,63,0.37684481143951415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,63,0.31250240802764895
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,1,0.6165408134460449
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,1,0.7483920097351074
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,3,0.7412799835205078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,3,0.6184463977813721
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,7,0.6218063831329346
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,15,0.7407792091369629
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,15,0.6169824123382568
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,1,0.014694400131702423
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,1,0.012555199861526489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,31,0.7488143920898438
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,3,0.014761599898338317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,31,0.6140063762664795
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,3,0.01252799928188324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,7,0.012511999905109405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,15,0.012547199428081513
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,31,0.014531199634075165
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,31,0.01252480000257492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,63,0.014638400077819825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,63,0.012470400333404541
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,127,0.014579200744628906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,127,0.012467200309038163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,255,0.015372799336910247
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,255,0.012508800625801087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,511,0.03508639931678772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,511,0.016607999801635742
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,1023,0.05767999887466431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,1023,0.033334401249885556
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,7,0.7419616222381592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,2047,0.10272159576416015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,2047,0.05625600218772888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,1,0.026846399903297423
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,3,0.026894399523735048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,1,0.022862400114536285
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,3,0.022771200537681578
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,7,0.022724799811840057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,7,0.014604799449443817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,15,0.026825600862503053
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,15,0.014636799693107605
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,31,0.026952001452445983
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,15,0.022968000173568724
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,31,0.022894400358200073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,63,0.022881600260734557
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,127,0.03210560083389282
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,127,0.024478399753570558
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,255,0.047276800870895384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,255,0.029105600714683533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,511,0.08166080117225646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,511,0.04731999933719635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,1023,0.15034400224685668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,1023,0.0836575984954834
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,2047,0.2876751899719238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,4095,0.5656288146972657
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,7,0.026921600103378296
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,1,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,4095,0.30452799797058105
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,63,0.02683520019054413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,3,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,2047,0.15648640394210817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,7,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,1,0.008761599659919739
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,3,0.008568000048398972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,15,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,15,0.008612799644470214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,31,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,31,0.008553600311279297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,63,0.010465600341558457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,63,0.008595199882984161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,127,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,127,0.00862559974193573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,255,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,255,0.00854720026254654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,511,0.008553600311279297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,1023,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,1023,0.008591999858617782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,2047,0.011449600011110306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,2047,0.010654400289058685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,4095,0.013352000713348388
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,4095,0.010569600015878677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,1,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,1,0.008553600311279297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,3,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,3,0.008452799916267396
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,7,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,15,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,15,0.008966399729251862
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,31,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,31,0.008548799902200699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,63,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,63,0.008897600322961807
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,127,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,127,0.010044799745082855
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,511,0.010585600137710571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,255,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,255,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,511,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,511,0.010375999659299851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,1023,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,1023,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,2047,0.012518399953842163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,2047,0.010896000266075134
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,4095,0.028905600309371948
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,4095,0.01451520025730133
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,1,0.016838400065898894
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,1,0.014718399941921234
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,3,0.017004799842834473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,3,0.014667199552059173
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,7,0.01690399944782257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,7,0.015913599729537965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,15,0.018646399676799773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,15,0.015974399447441102
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,31,0.018651199340820313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,31,0.01642719954252243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,63,0.018665599822998046
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,63,0.016519999504089354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,127,0.018695999681949616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,127,0.016438399255275727
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,255,0.028916800022125246
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,7,0.008556800335645676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,511,0.028579199314117433
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,1023,0.08071519732475281
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,1023,0.04602720141410828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,2047,0.14990240335464478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,2047,0.08240320086479187
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,4095,0.2905791997909546
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,7,0.008884800225496292
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,4095,0.15733120441436768
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,1,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,1,0.008441600203514098
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,3,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,3,0.008774399757385254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,7,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,7,0.008508799970149994
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,15,0.010632000118494033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,15,0.00846719965338707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,255,0.01573760062456131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,31,0.010603199899196624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,31,0.008463999629020691
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,63,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,511,0.04652160108089447
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,63,0.008636800199747085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,255,0.010454399883747101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,255,0.008659200370311737
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,511,0.00923679992556572
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,1023,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,2047,0.02885119915008545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,2047,0.012481600046157837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,4095,0.0453792005777359
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,4095,0.02701599895954132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,1,0.0452672004699707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,1,0.03907999992370605
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,3,0.04524320065975189
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,3,0.03728480041027069
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,7,0.04531840085983276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,127,0.008502399921417237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,511,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,1023,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,15,0.045300799608230594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,15,0.039427199959754945
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,31,0.03923999965190887
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,63,0.04949280023574829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,63,0.03927200138568878
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,127,0.0518127977848053
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,127,0.04010559916496277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,255,0.08194239735603333
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,255,0.04875519871711731
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,511,0.15002880096435547
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,511,0.08236640095710754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,7,0.038996800780296326
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,1023,0.15383679866790773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,31,0.047070398926734924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,2047,0.29955201148986815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,2047,0.5651696205139161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,1,0.0699072003364563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,3,0.08220319747924805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,3,0.06976959705352784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,7,0.08226400017738342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,7,0.06988639831542968
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,15,0.08440960049629212
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,31,0.08542240262031556
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,31,0.06990560293197631
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,63,0.08649759888648986
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,63,0.07000799775123596
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,127,0.09212480187416076
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,127,0.07208160161972046
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,255,0.1520159959793091
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,255,0.08612800240516663
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,1,0.08211359977722169
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,511,0.2879456043243408
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,1,0.1560655951499939
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,1,0.13125439882278442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,3,0.15605920553207397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,3,0.13144639730453492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,15,0.06864960193634033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,7,0.15828319787979125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,7,0.12929919958114625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,15,0.15807679891586304
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,15,0.1294800043106079
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,31,0.15835039615631102
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,63,0.1598047971725464
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,63,0.13146719932556153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,127,0.1708176016807556
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,127,0.13352639675140382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,511,0.1522096037864685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,255,0.29112958908081055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,1,0.010676799714565277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,255,0.16438239812850952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,1,0.010409600287675857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,3,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,7,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,7,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,15,0.010639999806880952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,15,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,31,0.011059200018644333
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,1023,0.287556791305542
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,31,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,63,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,127,0.010596799850463866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,127,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,255,0.010595200210809707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,255,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,511,0.01321599930524826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,511,0.010870400071144103
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,1023,0.02906239926815033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,1023,0.014604799449443817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,2047,0.047958400845527646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,2047,0.02892639935016632
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,4095,0.08120959997177124
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,3,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,4095,0.04749760031700134
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,1,0.2995728015899658
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,1,0.2532831907272339
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,3,0.2529616117477417
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,3,0.3021215915679932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,31,0.1294543981552124
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,7,0.3044912099838257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,63,0.01141119971871376
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,15,0.3016063928604126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,15,0.2541935920715332
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,31,0.30428640842437743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,31,0.25062720775604247
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,63,0.30755040645599363
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,63,0.2525583982467651
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,127,0.32447841167449953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,127,0.2569040060043335
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,1,0.597001600265503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,1,0.49434399604797363
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,3,0.6008416175842285
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,3,0.4969808101654053
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,7,0.5946752071380615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,7,0.4995296001434326
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,15,0.4941247940063477
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,7,0.25226559638977053
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,31,0.6017439842224122
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,31,0.49364800453186036
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,63,0.5990159988403321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,63,0.49377918243408203
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,1,0.013736000657081604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,3,0.012675200402736665
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,3,0.012489599734544754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,7,0.012587200105190276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,7,0.012507200241088867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,15,0.013804799318313599
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,15,0.012608000636100769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,31,0.013928000628948212
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,31,0.012587200105190276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,63,0.014508800208568573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,15,0.5970399856567383
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,63,0.012518399953842163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,127,0.013476799428462981
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,127,0.012548799812793731
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,255,0.013761599361896516
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,255,0.01255359947681427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,511,0.029183998703956604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,511,0.014577600359916686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,1023,0.04771519899368286
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,1023,0.02900800108909607
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,2047,0.08431199789047242
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,2047,0.047731199860572816
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,4095,0.0842095971107483
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,1,0.012617599964141846
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,1,0.02280319929122925
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,3,0.0227743998169899
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,1,0.018772800266742707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,7,0.022889600694179536
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,3,0.018769599497318268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,7,0.01886560022830963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,15,0.02279040068387985
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,15,0.01878879964351654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,31,0.022814400494098663
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,31,0.018812799453735353
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,63,0.02289759963750839
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,63,0.018801599740982056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,127,0.02489120066165924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,127,0.018756799399852753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,255,0.020776000618934632
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,255,0.03743999898433685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,511,0.06379520297050476
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,511,0.038134399056434634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,1023,0.06494719982147217
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,2047,0.22125918865203859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,2047,0.1194000005722046
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,4095,0.22767839431762696
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,4095,0.4321392059326172
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,4095,0.15561599731445314
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,3,0.010686399787664414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,1023,0.11726560592651367
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,7,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,1,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,1,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,15,0.010728000104427338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,15,0.008947200328111648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,3,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,31,0.00859839990735054
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,63,0.01077279970049858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,63,0.009670399874448777
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,127,0.01064319983124733
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,255,0.010809600353240967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,255,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,511,0.01061440035700798
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,511,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,1023,0.01080320030450821
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,1023,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,2047,0.01276479959487915
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,2047,0.01189119964838028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,7,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,4095,0.014577600359916686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,4095,0.012587200105190276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,1,0.01063840016722679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,1,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,3,0.010652799904346467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,3,0.010542400181293488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,7,0.010665600001811982
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,7,0.010686399787664414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,15,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,15,0.010564800351858139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,31,0.0112527996301651
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,31,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,63,0.010705599933862687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,63,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,31,0.010454399883747101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,127,0.01058719977736473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,127,0.010465600341558457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,255,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,511,0.011559999734163284
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,127,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,511,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,1023,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,1023,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,2047,0.012694400548934937
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,4095,0.01865279972553253
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,2047,0.012670400738716125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,4095,0.014070400595664978
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,1,0.016579200327396394
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,1,0.014667199552059173
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,3,0.01671359986066818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,3,0.014662399888038635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,7,0.016734400391578676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,7,0.014494399726390838
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,15,0.01661760061979294
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,15,0.014579200744628906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,31,0.016628800332546233
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,31,0.014644800126552582
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,63,0.01655679941177368
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,63,0.014609600603580474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,127,0.016675199568271636
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,127,0.014608000218868256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,255,0.018751999735832213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,255,0.014596800506114959
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,511,0.03930400013923645
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,511,0.01884319931268692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,1023,0.06605759859085084
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,1023,0.03883520066738129
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,2047,0.12082719802856445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,2047,0.06661760210990905
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,4095,0.230132794380188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,4095,0.12193119525909424
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,255,0.010545600205659866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,1,0.008638399839401244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,3,0.008603200316429138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,3,0.008387199789285659
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,7,0.00965920016169548
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,7,0.008566399663686752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,15,0.009283199906349182
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,15,0.00854720026254654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,31,0.01021919995546341
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,31,0.008579199761152267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,63,0.008540800213813782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,127,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,127,0.008544000238180161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,255,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,255,0.00854559987783432
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,511,0.010704000294208527
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,511,0.008694399893283845
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,1023,0.01255359947681427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,1023,0.010731200128793717
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,2047,0.020553599298000335
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,2047,0.012638400495052337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,1,0.008532799780368805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,4095,0.03781760036945343
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,4095,0.018827199935913086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,1,0.03723520040512085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,1,0.031113600730895995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,3,0.03712959885597229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,3,0.031195199489593504
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,7,0.03715519905090332
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,63,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,7,0.031020799279212953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,15,0.03719840049743652
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,15,0.031083199381828307
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,31,0.037264001369476316
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,31,0.03105440139770508
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,63,0.03766559958457947
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,63,0.031065601110458373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,127,0.04161120057106018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,127,0.03110080063343048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,255,0.06386719942092896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,255,0.03912000060081482
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,511,0.11499680280685425
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,511,0.06410080194473267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,1023,0.21806080341339112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,1023,0.11862239837646485
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,1,0.06365280151367188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,1,0.053504002094268796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,2047,0.2276384115219116
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,3,0.06365280151367188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,2047,0.42432317733764646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,3,0.053655999898910525
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,7,0.053491199016571046
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,7,0.06385599970817565
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,15,0.053604799509048465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,15,0.063755202293396
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,31,0.053855997323989865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,63,0.0677727997303009
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,127,0.07179999947547913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,127,0.05785920023918152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,255,0.11726880073547363
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,255,0.06690880060195922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,511,0.21924159526824952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,511,0.11769759654998779
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,1,0.11910400390625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,1,0.10062719583511352
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,3,0.11915839910507202
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,3,0.10060000419616699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,7,0.11961439847946168
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,15,0.12112640142440796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,15,0.10067839622497558
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,31,0.12136640548706054
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,31,0.10069279670715332
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,63,0.12184480428695679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,31,0.0658191978931427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,63,0.1010975956916809
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,63,0.053487998247146604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,127,0.12915680408477784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,127,0.10288480520248414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,255,0.2201551914215088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,255,0.12371679544448852
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,1,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,1,0.010393600165843963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,3,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,3,0.010391999781131745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,7,0.010595200210809707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,15,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,15,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,7,0.10071359872817993
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,31,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,63,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,63,0.010375999659299851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,127,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,127,0.008736000210046769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,255,0.010552000254392624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,255,0.008550400286912918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,511,0.012559999525547028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,511,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,1023,0.02001439929008484
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,1023,0.012630400061607362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,2047,0.04134719967842102
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,2047,0.020694400370121
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,4095,0.07146880030632019
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,7,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,4095,0.042561599612236024
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,1,0.22978720664978028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,1,0.19123680591583253
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,3,0.2298640012741089
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,31,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,7,0.1908079981803894
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,7,0.2315040111541748
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,15,0.2303839921951294
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,15,0.19300639629364014
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,31,0.2302464008331299
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,31,0.1930415987968445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,63,0.23249599933624268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,63,0.19316320419311522
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,127,0.24663360118865968
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,127,0.196124804019928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,1,0.4527552127838135
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,1,0.3728384017944336
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,3,0.449403190612793
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,3,0.37719199657440183
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,7,0.44709439277648927
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,7,0.3749727964401245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,15,0.4524256229400635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,3,0.19069600105285645
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,31,0.4509984016418457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,31,0.37705440521240235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,63,0.45008320808410646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,1,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,63,0.3758591890335083
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,3,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,7,0.012454400211572647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,7,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,15,0.012516799569129943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,15,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,31,0.012518399953842163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,31,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,63,0.012537600100040435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,63,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,127,0.012683199346065521
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,127,0.010708799958229065
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,255,0.012577599287033081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,255,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,511,0.01849920004606247
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,511,0.012617599964141846
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,1,0.012564800679683685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,1023,0.03776159882545471
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,1023,0.018611200153827667
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,2047,0.06401600241661072
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,4095,0.11685440540313721
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,4095,0.06502720117568969
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,1,0.016734400391578676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,1,0.0147024005651474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,3,0.016814400255680085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,3,0.014763200283050537
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,7,0.016809600591659545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,7,0.016359999775886536
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,15,0.018409599363803864
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,15,0.016590400040149687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,31,0.018615999817848207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,31,0.014664000272750855
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,63,0.01701280027627945
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,63,0.014611199498176575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,127,0.017118400335311888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,3,0.012492799758911132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,127,0.016527999937534333
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,255,0.02794719934463501
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,255,0.015833599865436553
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,511,0.0452208012342453
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,511,0.028790399432182312
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,1023,0.0801904022693634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,1023,0.04682720005512238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,2047,0.14994879961013793
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,2047,0.08505759835243225
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,4095,0.28940639495849607
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,15,0.3730047941207886
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,8191,0.5818064212799072
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,8191,0.30550079345703124
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,2047,0.03734399974346161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,1,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,3,0.010633599758148194
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,4095,0.15938559770584107
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,7,0.010704000294208527
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,1,0.0106175996363163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,3,0.009777600318193436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,15,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,15,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,31,0.010595200210809707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,31,0.008699200302362441
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,63,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,63,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,127,0.010155200213193893
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,255,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,255,0.010603199899196624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,511,0.010715200006961823
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,511,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,1023,0.010760000348091126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,1023,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,2047,0.011919999867677689
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,2047,0.010628800094127654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,4095,0.012724800407886505
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,4095,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,8191,0.014716799557209014
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,8191,0.012833599746227265
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,1,0.008788800239562989
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,1,0.008507200330495835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,3,0.008422400057315826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,3,0.008497600257396699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,7,0.008422400057315826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,7,0.008484800159931184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,15,0.008883199840784072
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,15,0.008475200086832047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,127,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,31,0.00862559974193573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,31,0.00846719965338707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,63,0.010302399843931198
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,127,0.008476799726486206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,127,0.008673600107431411
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,63,0.008462399989366532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,255,0.008771199733018875
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,511,0.008568000048398972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,511,0.008683200180530547
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,1023,0.010422399640083313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,1023,0.01058719977736473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,7,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,2047,0.010620799660682679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,4095,0.01273760050535202
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,4095,0.012518399953842163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,8191,0.028224000334739686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,8191,0.014585599303245544
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,1,0.012859199941158295
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,1,0.012508800625801087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,3,0.012700800597667695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,3,0.012520000338554382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,7,0.012736000120639801
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,7,0.012513600289821625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,15,0.012774400413036346
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,15,0.012444800138473511
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,31,0.012734399735927581
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,63,0.012695999443531036
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,63,0.0125231996178627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,127,0.012807999551296235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,127,0.012636800110340119
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,255,0.012654399871826172
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,255,0.013499200344085693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,511,0.02887359857559204
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,511,0.014747199416160584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,255,0.008390399813652038
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,1023,0.047366398572921756
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,1023,0.02884640097618103
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,2047,0.04635840058326721
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,2047,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,4095,0.15607199668884278
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,8191,0.29968159198760985
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,8191,0.15743039846420287
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,31,0.012516799569129943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,1,0.00984480008482933
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,1,0.008537600189447403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,3,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,3,0.010416000336408614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,7,0.010401599854230881
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,7,0.010355199873447418
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,15,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,15,0.010406400263309478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,31,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,31,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,63,0.009011200070381165
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,127,0.010390400141477584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,127,0.009228800237178803
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,255,0.010318399965763092
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,255,0.008745600283145905
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,2047,0.08224959969520569
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,511,0.008691199868917466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,4095,0.08414559960365295
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,1023,0.010739199817180634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,1023,0.010468800365924836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,2047,0.012671999633312225
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,2047,0.012147200107574464
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,4095,0.02897599935531616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,4095,0.014691199362277984
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,8191,0.045844799280166625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,8191,0.02768160104751587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,1,0.027059200406074523
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,1,0.02311680018901825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,63,0.010468800365924836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,3,0.027051201462745665
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,3,0.02279040068387985
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,7,0.027008000016212463
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,15,0.027035200595855714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,15,0.022972799837589264
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,31,0.026995199918746948
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,511,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,31,0.022945599257946016
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,63,0.026943999528884887
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,63,0.023003199696540834
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,127,0.0309935986995697
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,127,0.024864000082015992
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,255,0.04586879909038544
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,255,0.02951360046863556
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,511,0.08029919862747192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,511,0.04694879949092865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,1023,0.08191199898719788
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,2047,0.28748478889465334
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,2047,0.156278395652771
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,7,0.022788800299167633
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,4095,0.3025615930557251
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,4095,0.5676159858703613
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,1,0.04535839855670929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,1,0.039182400703430174
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,3,0.045337599515914914
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,7,0.03718880116939545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,15,0.045289599895477296
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,15,0.03710080087184906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,31,0.045296001434326175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,1023,0.1497007966041565
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,31,0.03715839982032776
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,63,0.04767360091209412
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,63,0.037294399738311765
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,127,0.05119519829750061
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,127,0.04038560092449188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,255,0.0487744003534317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,511,0.14950560331344603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,511,0.08223999738693237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,1023,0.28646399974823
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,1023,0.1516543984413147
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,1,0.08134400248527526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,3,0.03931359946727753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,1,0.06784800291061402
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,3,0.0815455973148346
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,7,0.08087520003318786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,7,0.06831039786338806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,15,0.08241440057754516
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,15,0.06999040246009827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,255,0.08038079738616943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,31,0.08404800295829773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,31,0.06996160149574279
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,63,0.08464959859848023
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,127,0.08985120058059692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,127,0.07108160257339477
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,255,0.15021439790725707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,255,0.08825759887695313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,7,0.04564160108566284
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,511,0.28707358837127683
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,511,0.15274080038070678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,1,0.00843840017914772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,3,0.008454400300979614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,1,0.008406399935483932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,3,0.008387199789285659
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,7,0.00960640013217926
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,15,0.010283199697732925
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,15,0.008591999858617782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,31,0.008916799724102021
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,31,0.008414400368928909
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,63,0.009459199756383896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,63,0.07028160095214844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,63,0.008548799902200699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,127,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,127,0.008532799780368805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,255,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,255,0.008615999668836593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,511,0.01058719977736473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,511,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,1023,0.012555199861526489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,1023,0.010636799782514573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,2047,0.027931201457977294
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,2047,0.0126351997256279
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,4095,0.04526239931583405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,4095,0.027318400144577027
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,8191,0.07981119751930237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,8191,0.04520640075206757
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,1,0.15589599609375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,1,0.12963199615478516
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,3,0.15586400032043457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,3,0.1298159956932068
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,7,0.15765119791030885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,7,0.1294432044029236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,15,0.15763039588928224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,7,0.008401600271463394
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,15,0.13136160373687744
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,31,0.1560912013053894
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,31,0.13148319721221924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,63,0.15664960145950318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,63,0.13234399557113646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,127,0.16773120164871216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,127,0.1338080048561096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,255,0.29043519496917725
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,255,0.16557760238647462
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,1,0.3003200054168701
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,1,0.2542959928512573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,3,0.30157759189605715
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,3,0.2542623996734619
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,3,0.06873760223388672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,7,0.3017296075820923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,7,0.2544719934463501
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,15,0.30204479694366454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,15,0.2521359920501709
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,31,0.30561439990997313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,31,0.2521552085876465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,63,0.303873610496521
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,63,0.25461440086364745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,127,0.32465760707855223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,127,0.2556688070297241
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,1,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,3,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,7,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,7,0.01048479974269867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,15,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,15,0.010196799784898758
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,31,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,31,0.009563200175762177
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,63,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,63,0.010000000149011612
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,127,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,255,0.010542400181293488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,127,0.010841599851846694
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,255,0.01043360009789467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,511,0.012929600477218629
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,1023,0.028811201453208923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,1023,0.014496000111103058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,2047,0.046305599808692935
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,1,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,2047,0.028889599442481994
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,3,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,4095,0.08036640286445618
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,4095,0.04640640020370483
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,1,0.016524800658226015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,8191,0.15455360412597657
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,8191,0.08440639972686767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,3,0.014611199498176575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,7,0.015457600355148315
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,511,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,15,0.01655520051717758
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,1,0.014662399888038635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,31,0.01664319932460785
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,31,0.014697599411010741
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,63,0.016702400147914888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,63,0.014694400131702423
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,3,0.014771200716495514
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,127,0.016451199352741242
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,127,0.014606399834156037
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,255,0.018838399648666383
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,255,0.014643199741840363
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,511,0.03932639956474304
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,511,0.020468799769878386
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,1023,0.03910079896450043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,2047,0.12056000232696533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,2047,0.06634079813957214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,4095,0.2294991970062256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,15,0.014472000300884247
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,4095,0.12310240268707276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,8191,0.2391632080078125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,8191,0.4492176055908203
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,1,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,3,0.010355199873447418
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,7,0.014528000354766845
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,7,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,1023,0.06595519781112671
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,15,0.010608000308275222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,1,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,3,0.010420800000429154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,31,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,63,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,7,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,63,0.01043199971318245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,127,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,127,0.010334400087594986
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,255,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,255,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,511,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,511,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,1023,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,15,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,2047,0.012427199631929398
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,2047,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,4095,0.012748800218105316
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,4095,0.012518399953842163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,8191,0.014603200554847717
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,8191,0.012758399546146392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,1,0.010622400045394897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,1,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,3,0.010660800337791442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,3,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,7,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,7,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,15,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,15,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,31,0.010639999806880952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,31,0.0104032002389431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,63,0.010654400289058685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,63,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,1023,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,127,0.010678400099277497
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,127,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,255,0.011553599685430526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,511,0.010620799660682679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,511,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,1023,0.011456000059843064
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,1023,0.010742399841547012
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,2047,0.012806400656700134
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,2047,0.012489599734544754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,4095,0.014641599357128143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,4095,0.012700800597667695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,31,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,8191,0.018915200233459474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,8191,0.014710399508476257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,1,0.01067200005054474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,3,0.012673600018024445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,3,0.010713600367307664
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,7,0.012606400251388549
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,15,0.012537600100040435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,7,0.011023999750614166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,15,0.010711999982595444
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,255,0.00992320030927658
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,31,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,63,0.012561599910259246
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,63,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,127,0.012465599924325943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,127,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,255,0.012540799379348756
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,255,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,511,0.01849920004606247
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,511,0.012606400251388549
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,1023,0.0186256006360054
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,2047,0.06372960209846497
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,2047,0.03734720051288605
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,4095,0.11636960506439209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,4095,0.06449120044708252
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,1,0.012491200119256973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,8191,0.2244784116744995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,8191,0.12057119607925415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,1,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,3,0.01040479987859726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,3,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,7,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,7,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,15,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,31,0.012758399546146392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,15,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,31,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,63,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,63,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,1023,0.03710080087184906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,127,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,255,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,255,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,511,0.010422399640083313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,511,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,1023,0.012441600114107132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,1023,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,2047,0.012588800489902496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,2047,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,4095,0.018681600689888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,4095,0.012627199292182922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,8191,0.039099198579788205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,8191,0.01780479997396469
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,1,0.010326399654150008
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,1,0.022767999768257143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,1,0.020423999428749083
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,3,0.022752000391483305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,3,0.019249600172042847
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,7,0.022716799378395082
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,7,0.02062560021877289
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,15,0.02065120041370392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,31,0.022814400494098663
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,31,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,31,0.020587199926376344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,63,0.022779199481010436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,63,0.019713599979877473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,127,0.024860799312591553
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,127,0.019465599954128266
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,127,0.010424000024795533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,255,0.03808799982070923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,511,0.06381760239601135
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,511,0.03872160017490387
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,1023,0.11621439456939697
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,1023,0.0657039999961853
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,2047,0.22129759788513184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,2047,0.11990560293197632
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,4095,0.4320208072662354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,15,0.02271360009908676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,1,0.03650240004062653
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,4095,0.22824640274047853
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,1,0.03112959861755371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,3,0.035158398747444156
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,3,0.03094879984855652
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,7,0.0350383996963501
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,7,0.030985599756240843
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,15,0.03557440042495728
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,15,0.031016001105308534
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,31,0.036396801471710205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,31,0.030959999561309813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,63,0.039243200421333314
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,63,0.031204798817634584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,127,0.03302719891071319
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,255,0.06378719806671143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,255,0.039113599061965945
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,511,0.11516000032424926
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,511,0.06382079720497132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,1023,0.2182447910308838
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,1023,0.11868480443954468
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,1,0.06367840170860291
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,3,0.06370880007743836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,1,0.05354560017585754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,3,0.05351520180702209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,7,0.0639840006828308
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,7,0.053585600852966306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,15,0.06393120288848878
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,255,0.020761600136756896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,31,0.06709280014038085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,31,0.05363360047340393
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,63,0.06794400215148926
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,127,0.041633599996566774
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,127,0.07147200107574463
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,127,0.05578399896621704
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,255,0.11647839546203613
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,511,0.21890881061553955
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,511,0.11801760196685791
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,1,0.008641599863767623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,1,0.008580800145864487
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,3,0.008534400165081025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,3,0.00867839977145195
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,7,0.009939199686050415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,7,0.008499199897050858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,15,0.010367999970912933
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,15,0.008535999804735184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,31,0.010375999659299851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,31,0.008590400218963623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,63,0.0554751992225647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,63,0.010375999659299851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,63,0.008392000198364257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,127,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,127,0.008457600325345992
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,255,0.010318399965763092
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,255,0.06792479753494263
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,511,0.010564800351858139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,511,0.008369600027799606
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,1023,0.012460800260305405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,1023,0.01040479987859726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,2047,0.019176000356674196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,2047,0.01252799928188324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,4095,0.0386000007390976
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,4095,0.018632000684738158
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,8191,0.0364544004201889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,1,0.11736639738082885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,1,0.10053600072860717
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,3,0.1178879976272583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,3,0.10058879852294922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,7,0.11914399862289429
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,7,0.10055999755859375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,15,0.05351679921150208
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,15,0.11999679803848266
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,31,0.12044960260391235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,31,0.10090399980545044
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,255,0.008555199950933456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,63,0.10270559787750244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,127,0.1298416018486023
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,127,0.1047551989555359
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,8191,0.06440479755401611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,255,0.2213887929916382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,255,0.12672159671783448
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,1,0.22989280223846437
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,1,0.19277280569076538
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,3,0.23037118911743165
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,3,0.19268959760665894
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,7,0.23161120414733888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,7,0.19287840127944947
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,15,0.23362879753112792
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,15,0.1927888035774231
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,31,0.23193440437316895
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,31,0.19296799898147582
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,63,0.12125279903411865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,63,0.23279359340667724
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,63,0.19455360174179076
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,127,0.24663519859313965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,127,0.19777120351791383
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,1,0.00849440023303032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,3,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,3,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,7,0.010662399977445603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,7,0.010331200063228607
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,15,0.010606399923563003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,15,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,31,0.010728000104427338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,31,0.010639999806880952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,15,0.10055359601974487
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,63,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,127,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,127,0.010328000038862228
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,255,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,255,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,511,0.01257600039243698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,511,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,1023,0.0189968004822731
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,1,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,1023,0.012801599502563477
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,2047,0.018694399297237395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,2047,0.041387200355529785
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,4095,0.07114880084991455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,4095,0.04187360107898712
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,8191,0.13068000078201295
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,8191,0.07576320171356202
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,1,0.012668800354003907
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,3,0.012545600533485413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,1,0.012585599720478059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,7,0.012812800705432892
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,7,0.012510399520397186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,15,0.012721599638462066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,63,0.010728000104427338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,15,0.012547199428081513
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,31,0.012718400359153748
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,31,0.012478400021791458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,63,0.012668800354003907
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,63,0.012535999715328216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,127,0.012681600451469422
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,127,0.012520000338554382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,255,0.012763200700283051
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,255,0.012571200728416443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,511,0.028300800919532777
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,511,0.014766399562358857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,1023,0.04542239904403687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,2047,0.08099200129508972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,1023,0.029473599791526795
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,4095,0.1518399953842163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,4095,0.08502879738807678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,8191,0.29408481121063235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,8191,0.1592303991317749
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,3,0.012567999958992004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,1,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,16383,0.31472480297088623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,16383,0.5791759967803956
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,3,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,7,0.010732799768447876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,2047,0.04683679938316345
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,15,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,1,0.010632000118494033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,3,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,31,0.012913599610328674
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,31,0.010406400263309478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,7,0.010635200142860412
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,63,0.01043199971318245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,127,0.010371199995279311
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,255,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,255,0.010353600233793258
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,511,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,511,0.010391999781131745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,1023,0.010449600219726563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,1023,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,2047,0.010742399841547012
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,15,0.008444800227880477
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,4095,0.012577599287033081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,4095,0.012577599287033081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,8191,0.014638400077819825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,8191,0.012673600018024445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,16383,0.016569599509239197
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,16383,0.014611199498176575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,1,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,1,0.009587199985980987
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,3,0.010606399923563003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,3,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,63,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,127,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,15,0.010545600205659866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,15,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,31,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,31,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,63,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,2047,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,63,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,127,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,255,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,255,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,511,0.010628800094127654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,511,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,1023,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,1023,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,2047,0.010673599690198899
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,7,0.010567999631166457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,7,0.00923840031027794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,4095,0.012742400169372559
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,4095,0.011987199634313583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,8191,0.014812800288200378
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,8191,0.012809599936008453
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,127,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,16383,0.01656640022993088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,1,0.010659199953079224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,1,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,3,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,3,0.010367999970912933
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,7,0.010372799634933472
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,7,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,15,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,15,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,31,0.010620799660682679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,31,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,63,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,63,0.011128000169992446
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,127,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,127,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,255,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,255,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,16383,0.02959359884262085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,511,0.012751999497413635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,511,0.010728000104427338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,1023,0.027236801385879517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,1023,0.01475519984960556
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,2047,0.0454367995262146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,2047,0.028494399785995484
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,4095,0.07998560070991516
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,4095,0.0472544014453888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,8191,0.14940639734268188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,16383,0.2870192050933838
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,16383,0.1620800018310547
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,1,0.010100799798965453
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,2047,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,3,0.010000000149011612
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,3,0.008459199965000153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,7,0.009961599856615067
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,7,0.008473599702119828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,15,0.010158400237560272
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,15,0.008462399989366532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,31,0.008798400312662125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,63,0.00888800024986267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,63,0.008908800035715102
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,127,0.00881119966506958
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,127,0.008961600065231324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,255,0.010340800136327743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,255,0.008806400001049042
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,511,0.009908799827098847
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,511,0.00883840024471283
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,8191,0.08442080020904541
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,1023,0.010705599933862687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,1023,0.0084927998483181
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,2047,0.010777600109577179
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,2047,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,4095,0.013468800485134125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,4095,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,1,0.00862239971756935
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,8191,0.014486399292945863
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,16383,0.04552319943904877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,16383,0.027798399329185486
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,1,0.016864000260829924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,1,0.01467359960079193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,3,0.016771200299263
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,3,0.014593599736690522
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,7,0.01687519997358322
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,31,0.008470399677753449
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,15,0.017126399278640746
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,15,0.01462559998035431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,31,0.01716960072517395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,31,0.014609600603580474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,63,0.018654400110244752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,63,0.014620800316333771
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,127,0.018142400681972502
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,127,0.014542399346828461
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,255,0.027356800436973572
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,255,0.014556799829006196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,511,0.04545120000839233
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,511,0.027825599908828734
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,1023,0.046107199788093564
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,2047,0.14775680303573607
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,2047,0.08183680176734924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,4095,0.2858016014099121
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,8191,0.02720640003681183
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,4095,0.15644320249557495
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,7,0.014727999269962311
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,8191,0.3088079929351807
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,1,0.027011200785636902
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,1,0.022784000635147093
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,3,0.022753599286079406
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,7,0.02685439884662628
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,7,0.022737599909305573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,15,0.026873600482940675
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,15,0.022726400196552275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,1023,0.08007519841194152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,31,0.02690559923648834
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,31,0.02289759963750839
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,63,0.02683520019054413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,63,0.022758400440216063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,127,0.029919999837875366
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,127,0.022833600640296936
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,255,0.045284798741340636
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,255,0.028907200694084166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,511,0.07983840107917786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,511,0.046526399254798886
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,1023,0.14778399467468262
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,8191,0.5608240127563476
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,1023,0.08172159790992736
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,3,0.02691679894924164
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,2047,0.2852992057800293
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,2047,0.15308159589767456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,1,0.038996800780296326
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,3,0.04529440104961395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,3,0.03898879885673523
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,7,0.03905119895935059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,7,0.04533120095729828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,15,0.045310398936271666
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,15,0.03810720145702362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,31,0.04539999961853027
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,31,0.039238399267196654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,63,0.03920640051364899
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,127,0.04999839961528778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,127,0.04139840006828308
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,255,0.08032320141792297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,255,0.049460801482200625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,511,0.14820640087127684
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,511,0.08208159804344177
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,1,0.04601120054721832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,1023,0.28549439907073976
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,1,0.010694400221109391
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,1,0.008369600027799606
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,1023,0.15292320251464844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,3,0.008478400111198426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,3,0.010675200074911118
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,7,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,15,0.01037919968366623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,63,0.04741120040416717
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,31,0.010273600369691849
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,63,0.010300800204277039
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,63,0.010393600165843963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,127,0.010313600301742554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,127,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,255,0.010358399897813796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,255,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,511,0.010366400331258773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,511,0.009118399769067764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,1023,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,1023,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,2047,0.012689599394798278
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,2047,0.011771199852228164
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,4095,0.02818560004234314
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,4095,0.014382399618625641
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,8191,0.045342400670051575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,8191,0.02820639908313751
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,16383,0.07973600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,16383,0.0454479992389679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,1,0.08207679986953735
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,7,0.008404800295829773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,1,0.06988319754600525
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,15,0.008551999926567078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,3,0.08209279775619507
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,3,0.06986879706382751
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,31,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,7,0.06788319945335389
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,15,0.08239679932594299
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,31,0.08231520056724548
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,31,0.06993119716644287
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,63,0.08420000076293946
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,63,0.07148640155792237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,127,0.09004160165786743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,127,0.07311360239982605
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,255,0.1505120038986206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,255,0.09049599766731262
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,511,0.2870415925979614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,511,0.15282080173492432
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,1,0.15470080375671386
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,1,0.1314288020133972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,3,0.15418720245361328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,3,0.13137600421905518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,7,0.15579040050506593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,7,0.1314079999923706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,15,0.15607999563217162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,15,0.13153120279312133
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,31,0.15622400045394896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,31,0.13151040077209472
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,63,0.15805280208587646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,63,0.1328336000442505
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,7,0.08218240141868591
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,127,0.1686992049217224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,15,0.0703760027885437
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,1,0.008668799698352814
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,255,0.2910559892654419
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,255,0.1712767958641052
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,1,0.008611200004816055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,3,0.008497600257396699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,3,0.008406399935483932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,7,0.008601599931716919
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,15,0.00841279998421669
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,7,0.008584000170230865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,15,0.008428800106048583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,31,0.00865280032157898
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,31,0.008481600135564805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,63,0.008931200206279754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,63,0.008420799672603608
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,127,0.00862239971756935
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,255,0.00862559974193573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,255,0.008657599985599517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,511,0.010603199899196624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,511,0.008755200356245042
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,1023,0.012742400169372559
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,1023,0.01061440035700798
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,2047,0.02707040011882782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,2047,0.01268640011548996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,4095,0.044838398694992065
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,127,0.1353327989578247
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,4095,0.027163198590278624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,8191,0.0787551999092102
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,8191,0.04542079865932465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,16383,0.14791840314865112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,16383,0.08060640096664429
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,1,0.010686399787664414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,3,0.010582400113344192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,127,0.008724799752235413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,1,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,7,0.010449600219726563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,7,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,3,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,15,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,31,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,31,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,63,0.010796800255775452
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,63,0.010262399911880493
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,127,0.010708799958229065
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,127,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,255,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,255,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,511,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,511,0.010713600367307664
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,1023,0.014632000029087067
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,2047,0.0452320009469986
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,2047,0.02834239900112152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,4095,0.0802784025669098
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,4095,0.04585599899291992
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,8191,0.14958080053329467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,8191,0.08220959901809692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,16383,0.1533951997756958
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,16383,0.28877921104431153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,32767,0.2946831941604614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,32767,0.5665200233459473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,1,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,3,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,1023,0.02707839906215668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,7,0.010396800190210342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,15,0.010396800190210342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,1,0.010382399708032609
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,15,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,15,0.010334400087594986
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,31,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,31,0.010407999902963639
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,63,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,127,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,127,0.00873439982533455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,255,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,255,0.009190399944782258
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,7,0.010416000336408614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,511,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,511,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,1023,0.01063840016722679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,1023,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,2047,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,4095,0.012583999335765839
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,4095,0.012561599910259246
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,8191,0.014564800262451171
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,8191,0.014801600575447082
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,16383,0.016603200137615202
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,16383,0.016420799493789672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,3,0.00856959968805313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,32767,0.018963199853897095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,32767,0.016657599806785585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,1,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,1,0.008488000184297562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,3,0.010659199953079224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,3,0.008457600325345992
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,7,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,7,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,15,0.010390400141477584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,31,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,31,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,63,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,63,0.009935999661684037
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,2047,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,127,0.010236799716949463
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,255,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,255,0.01037919968366623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,511,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,511,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,1023,0.010366400331258773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,63,0.010454399883747101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,1023,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,15,0.010401599854230881
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,2047,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,4095,0.012582400441169738
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,4095,0.010791999846696853
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,8191,0.012606400251388549
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,8191,0.012532800436019897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,127,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,16383,0.01568319946527481
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,16383,0.014708800613880158
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,32767,0.016667200624942778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,1,0.008664000034332275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,1,0.008531200140714646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,3,0.008516799658536911
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,3,0.008497600257396699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,7,0.008534400165081025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,7,0.008529599756002426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,15,0.008531200140714646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,15,0.008459199965000153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,31,0.00851840004324913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,31,0.008499199897050858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,63,0.008558399975299835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,63,0.008495999872684479
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,127,0.008510400354862214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,127,0.008499199897050858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,255,0.00852160006761551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,255,0.008432000130414962
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,511,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,511,0.010409600287675857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,1023,0.012414400279521943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,32767,0.031036800146102904
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,1023,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,2047,0.0268095999956131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,4095,0.04467360079288483
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,4095,0.026899200677871705
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,8191,0.07926399707794189
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,8191,0.04522559940814972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,16383,0.14801440238952637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,16383,0.07982239723205567
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,32767,0.2868623971939087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,2047,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,1,0.010382399708032609
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,32767,0.14972959756851195
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,3,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,3,0.01037919968366623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,7,0.010465600341558457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,7,0.008680000156164169
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,15,0.010409600287675857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,15,0.00860000029206276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,31,0.010435199737548828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,31,0.008504000306129456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,63,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,2047,0.01252480000257492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,127,0.010449600219726563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,63,0.009494400024414063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,127,0.010369600355625152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,255,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,255,0.010420800000429154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,511,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,511,0.010444799810647965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,1023,0.010454399883747101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,2047,0.01143999993801117
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,2047,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,4095,0.012630400061607362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,4095,0.01241919994354248
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,8191,0.01459839940071106
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,8191,0.013500800728797913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,16383,0.029912000894546507
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,16383,0.016547200083732606
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,1,0.00867680013179779
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,32767,0.047260800004005434
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,1,0.012537600100040435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,1,0.012352000176906585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,3,0.012596799433231354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,3,0.012443199753761292
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,7,0.012625600397586822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,15,0.012644800543785095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,7,0.012449599802494049
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,15,0.012548799812793731
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,31,0.012615999579429627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,31,0.012449599802494049
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,63,0.012644800543785095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,63,0.012505599856376648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,127,0.012656000256538392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,127,0.012444800138473511
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,255,0.012624000012874604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,255,0.012433599680662155
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,1023,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,511,0.027900800108909607
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,511,0.014686399698257446
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,1023,0.04540160000324249
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,1023,0.028971201181411742
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,2047,0.04678399860858917
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,4095,0.081905597448349
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,8191,0.29391839504241946
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,8191,0.15158560276031494
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,32767,0.030513599514961243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,16383,0.293287992477417
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,16383,0.5786479949951172
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,1,0.01659200042486191
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,1,0.01451839953660965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,3,0.016606399416923524
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,3,0.014628799259662628
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,7,0.01684959977865219
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,7,0.014871999621391296
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,15,0.016646400094032288
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,15,0.015512000024318694
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,2047,0.08155679702758789
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,31,0.01666080057621002
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,31,0.014679999649524688
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,4095,0.15211839675903321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,63,0.014619199931621552
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,127,0.018503999710083006
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,255,0.027134400606155396
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,255,0.016499200463294984
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,511,0.04519520103931427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,511,0.028708800673484802
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,1023,0.07993280291557311
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,1023,0.0463375985622406
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,2047,0.14796960353851318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,2047,0.08118720054626465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,4095,0.2855184078216553
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,63,0.01661760061979294
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,1,0.02680320143699646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,127,0.01478559970855713
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,1,0.022711999714374542
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,3,0.02688480019569397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,3,0.022729599475860597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,7,0.027084800601005554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,15,0.027156800031661987
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,15,0.024191999435424806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,31,0.02738719880580902
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,31,0.02285760045051575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,63,0.02691200077533722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,63,0.02289759963750839
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,127,0.030745598673820495
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,127,0.022852799296379088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,255,0.04547840058803558
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,255,0.029049599170684816
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,511,0.07996479868888855
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,511,0.047219198942184445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,4095,0.1499951958656311
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,2047,0.28556480407714846
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,7,0.022801600396633148
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,1,0.008379200100898742
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,2047,0.15020639896392823
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,1,0.00838399976491928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,3,0.00846880003809929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,7,0.008481600135564805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,7,0.008371199667453765
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,15,0.008556800335645676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,15,0.008379200100898742
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,31,0.008459199965000153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,31,0.00836160033941269
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,63,0.008350399881601333
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,63,0.00835840031504631
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,127,0.00846560001373291
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,127,0.008432000130414962
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,255,0.00843520015478134
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,255,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,511,0.008441600203514098
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,511,0.008475200086832047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,1023,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,1023,0.08145599961280822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,2047,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,2047,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,4095,0.01266240030527115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,4095,0.0106175996363163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,8191,0.02799839973449707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,8191,0.014449599385261535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,16383,0.04533919990062714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,16383,0.028505599498748778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,32767,0.08023999929428101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,32767,0.04635840058326721
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,1,0.045307201147079465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,1,0.03919520080089569
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,3,0.04538559913635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,3,0.03915199935436249
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,7,0.04518879950046539
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,7,0.03916960060596466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,15,0.04532159864902496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,15,0.03916319906711578
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,31,0.045291200280189514
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,31,0.03920960128307342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,63,0.047577598690986635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,63,0.03917120099067688
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,127,0.05063040256500244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,127,0.04118559956550598
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,3,0.009683199971914292
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,255,0.04952479898929596
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,511,0.14871360063552858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,511,0.08222879767417908
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,1023,0.28558878898620604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,1023,0.15122400522232055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,1,0.08216480016708375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,1023,0.008959999680519104
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,1,0.06989759802818299
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,3,0.08219360113143921
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,3,0.07050079703330994
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,7,0.0821183979511261
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,7,0.0699184000492096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,15,0.08225439786911011
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,15,0.06999359726905822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,255,0.08014559745788574
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,31,0.08263360261917115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,31,0.06988319754600525
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,63,0.0841488003730774
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,63,0.07193120121955872
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,127,0.09023680090904236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,127,0.07367519736289978
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,255,0.1503327965736389
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,255,0.0883247971534729
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,511,0.15275360345840455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,511,0.28717119693756105
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,1,0.010395199805498124
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,1,0.008667200058698653
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,3,0.01032480001449585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,3,0.008484800159931184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,7,0.010302399843931198
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,7,0.0083856001496315
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,15,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,15,0.00846719965338707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,1023,0.1480512022972107
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,31,0.010416000336408614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,31,0.008470399677753449
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,63,0.01048479974269867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,127,0.010262399911880493
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,255,0.010316800326108932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,255,0.008499199897050858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,511,0.010340800136327743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,511,0.008460800349712371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,1023,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,1023,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,2047,0.012587200105190276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,2047,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,4095,0.027270400524139406
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,4095,0.014504000544548035
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,8191,0.04535680115222931
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,8191,0.02871519923210144
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,16383,0.08010879755020142
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,16383,0.04551520049571991
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,32767,0.14825439453125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,63,0.010407999902963639
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,32767,0.0806111991405487
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,1,0.008399999886751174
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,3,0.008376000076532364
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,1,0.008603200316429138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,7,0.008564800024032593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,15,0.009401600062847137
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,7,0.008451200276613235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,31,0.009286399930715561
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,31,0.008462399989366532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,63,0.009726399928331375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,63,0.008484800159931184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,127,0.00859680026769638
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,127,0.008782400190830231
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,255,0.008470399677753449
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,255,0.008553600311279297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,511,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,511,0.010390400141477584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,1023,0.012457600235939026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,1023,0.010688000172376633
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,2047,0.026977598667144775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,127,0.00992320030927658
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,2047,0.012656000256538392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,4095,0.044095999002456664
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,4095,0.027270400524139406
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,8191,0.07796800136566162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,8191,0.04534879922866821
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,3,0.008403199911117553
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,16383,0.1465456008911133
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,16383,0.07987679839134217
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,15,0.008627200126647949
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,32767,0.28390719890594485
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,32767,0.14882880449295044
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,1,0.010375999659299851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,3,0.008499199897050858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,65535,0.2878031969070435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,65535,0.5552847862243653
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,7,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,15,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,1,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,3,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,31,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,31,0.008524800091981888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,63,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,63,0.008526399731636047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,127,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,127,0.00846880003809929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,255,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,255,0.008511999994516373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,7,0.008491200208663941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,511,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,511,0.00846560001373291
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,1023,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,2047,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,2047,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,4095,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,4095,0.011830399930477142
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,8191,0.012628799676895142
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,8191,0.012563200294971466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,16383,0.018695999681949616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,16383,0.016729600727558136
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,32767,0.018691200017929076
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,32767,0.018563200533390046
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,65535,0.02067359983921051
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,65535,0.018801599740982056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,1,0.009806399792432785
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,1,0.00851840004324913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,3,0.009841600060462951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,3,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,7,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,7,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,15,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,15,0.008460800349712371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,31,0.009612800180912017
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,1023,0.010542400181293488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,63,0.010337600111961364
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,63,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,127,0.010440000146627427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,127,0.010344000160694122
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,255,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,255,0.009736000001430512
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,511,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,511,0.008532799780368805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,1023,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,1023,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,2047,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,2047,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,4095,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,4095,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,8191,0.014558400213718414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,8191,0.013548800349235534
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,16383,0.016599999368190767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,32767,0.018615999817848207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,16383,0.016920000314712524
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,32767,0.016612799465656282
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,31,0.008508799970149994
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,65535,0.018624000251293182
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,1,0.008588799834251403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,1,0.008430399745702744
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,3,0.008531200140714646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,3,0.008483199775218964
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,7,0.008457600325345992
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,7,0.008638399839401244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,15,0.009294400364160538
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,31,0.009299200028181076
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,31,0.008392000198364257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,63,0.008879999816417693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,63,0.008472000062465668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,127,0.00968960002064705
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,127,0.008484800159931184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,255,0.008752000331878663
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,255,0.00846560001373291
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,511,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,511,0.009265600144863129
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,1023,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,1023,0.010424000024795533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,2047,0.012540799379348756
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,2047,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,4095,0.028003200888633728
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,4095,0.014511999487876893
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,8191,0.04498879909515381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,15,0.010356800258159637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,8191,0.027496001124382018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,16383,0.08009920120239258
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,16383,0.04528320133686066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,65535,0.035355201363563536
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,32767,0.08011360168457031
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,15,0.00846880003809929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,1,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,1,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,3,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,3,0.01024319976568222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,7,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,7,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,15,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,15,0.010398399829864503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,31,0.010380800068378448
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,31,0.008451200276613235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,63,0.009489600360393525
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,63,0.008463999629020691
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,127,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,127,0.009286399930715561
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,255,0.010011199861764908
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,255,0.008705600351095199
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,511,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,511,0.010420800000429154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,1023,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,1023,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,2047,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,2047,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,4095,0.012505599856376648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,4095,0.010862399637699128
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,65535,0.2842400074005127
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,65535,0.14969120025634766
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,8191,0.014239999651908874
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,16383,0.01663679927587509
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,16383,0.014678399264812469
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,32767,0.03113600015640259
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,32767,0.016542400419712066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,65535,0.04892959892749786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,65535,0.03139199912548065
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,1,0.010444799810647965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,1,0.010339199751615524
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,3,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,3,0.009836799651384353
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,7,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,7,0.01040000021457672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,15,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,15,0.01040000021457672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,31,0.010391999781131745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,31,0.010471999645233154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,63,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,63,0.010036800056695938
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,32767,0.14755359888076783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,127,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,255,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,127,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,8191,0.012721599638462066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,255,0.010420800000429154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,511,0.012691199779510498
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,511,0.01091040000319481
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,1023,0.0268095999956131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,1023,0.014584000408649444
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,2047,0.04525440037250519
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,2047,0.028167998790740965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,4095,0.08034080266952515
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,4095,0.046137601137161255
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,8191,0.14795520305633544
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,8191,0.08184959888458251
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,16383,0.2860975980758667
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,16383,0.1535264015197754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,1,0.012505599856376648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,1,0.012880000472068786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,32767,0.5627711772918701
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,3,0.012486399710178375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,32767,0.29238879680633545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,7,0.012457600235939026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,3,0.012484800070524216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,7,0.012513600289821625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,15,0.012508800625801087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,31,0.012486399710178375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,31,0.012521600723266602
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,63,0.01244639977812767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,127,0.012638400495052337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,255,0.012571200728416443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,255,0.012488000094890594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,511,0.028270399570465087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,511,0.014739200472831726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,1023,0.045484799146652224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,1023,0.029017600417137145
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,2047,0.08152639865875244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,2047,0.04738720059394837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,4095,0.15184799432754517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,4095,0.08221759796142578
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,8191,0.29287359714508054
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,15,0.012521600723266602
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,1,0.01653600037097931
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,8191,0.15282880067825316
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,1,0.014632000029087067
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,3,0.016681599617004394
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,3,0.01656160056591034
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,7,0.016700799763202667
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,7,0.016595199704170227
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,15,0.01671359986066818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,15,0.01663679927587509
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,127,0.012539200484752655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,31,0.01674720048904419
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,31,0.01656640022993088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,63,0.01675360053777695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,63,0.016553600132465363
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,127,0.016468800604343414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,255,0.026958400011062623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,255,0.016441600024700166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,511,0.028838399052619933
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,1023,0.07887840270996094
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,1023,0.04604319930076599
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,2047,0.14726239442825317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,2047,0.08132799863815307
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,4095,0.2844543933868408
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,4095,0.150654399394989
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,63,0.01263359934091568
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,1,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,1,0.008404800295829773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,3,0.008528000116348267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,7,0.010366400331258773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,7,0.00878560021519661
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,127,0.016755199432373045
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,15,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,15,0.009198399633169175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,31,0.010470400005578995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,31,0.009078399837017059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,511,0.04524799883365631
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,63,0.008947200328111648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,127,0.010356800258159637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,127,0.009470400214195252
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,255,0.010393600165843963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,255,0.009603200107812881
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,511,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,511,0.010449600219726563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,1023,0.01053439974784851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,1023,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,2047,0.011374399811029435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,2047,0.010609599947929382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,4095,0.012611199915409089
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,4095,0.012412799894809723
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,8191,0.014679999649524688
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,8191,0.012569600343704223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,16383,0.02980799973011017
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,16383,0.016516800224781036
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,3,0.010422399640083313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,32767,0.047307199239730834
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,32767,0.03089439868927002
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,65535,0.08109440207481385
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,65535,0.04845280051231384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,1,0.026878398656845093
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,1,0.0247871994972229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,3,0.026947200298309326
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,63,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,3,0.024777600169181825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,7,0.024728000164031982
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,15,0.026878398656845093
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,15,0.023524799942970277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,31,0.026919999718666078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,31,0.02391359955072403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,63,0.026955199241638184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,63,0.02287680059671402
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,127,0.030803200602531434
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,127,0.023636800050735474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,255,0.045326399803161624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,511,0.0800432026386261
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,511,0.04742720127105713
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,1023,0.14806560277938843
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,1023,0.08220959901809692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,2047,0.28383359909057615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,2047,0.150108802318573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,1,0.04541119933128357
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,7,0.026939201354980468
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,1,0.03924799859523773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,3,0.04521760046482086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,3,0.04097920060157776
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,7,0.03911679983139038
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,15,0.04535520076751709
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,15,0.03913759887218475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,31,0.045265600085258484
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,31,0.03909600079059601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,63,0.04749439954757691
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,255,0.029507198929786684
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,127,0.04987519979476929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,127,0.04288640022277832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,255,0.08013120293617249
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,255,0.04955680072307587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,511,0.1486400008201599
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,511,0.08223680257797242
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,1023,0.28499999046325686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,1023,0.1518944025039673
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,1,0.009676799923181535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,1,0.008428800106048583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,3,0.00957759991288185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,3,0.008504000306129456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,7,0.00950239971280098
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,7,0.00849440023303032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,15,0.01005920022726059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,15,0.008422400057315826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,31,0.010075200349092484
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,31,0.00840959995985031
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,63,0.00843520015478134
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,127,0.008403199911117553
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,127,0.008422400057315826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,7,0.04532159864902496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,255,0.008489599823951722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,255,0.008537600189447403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,511,0.008481600135564805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,511,0.008550400286912918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,1023,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,1023,0.008667200058698653
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,2047,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,2047,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,4095,0.012515200674533844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,4095,0.01223360002040863
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,8191,0.027132800221443175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,8191,0.014542399346828461
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,16383,0.045054399967193605
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,16383,0.02860960066318512
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,32767,0.07813439965248108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,32767,0.04614399969577789
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,65535,0.14649440050125123
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,1,0.008457600325345992
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,63,0.009987200051546097
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,3,0.009964799880981446
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,63,0.03919520080089569
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,7,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,65535,0.08114879727363586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,15,0.010345599800348281
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,1,0.008539199829101562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,31,0.008462399989366532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,31,0.008537600189447403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,63,0.008502399921417237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,3,0.008513599634170532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,63,0.009812799841165542
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,127,0.010555200278759003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,127,0.009991999715566635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,255,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,255,0.009372799843549728
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,7,0.008696000277996063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,511,0.010424000024795533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,511,0.010147199779748917
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,1023,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,1023,0.0107744000852108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,2047,0.01255040019750595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,2047,0.012520000338554382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,4095,0.0270687997341156
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,4095,0.014686399698257446
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,15,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,8191,0.04528320133686066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,8191,0.028963199257850646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,16383,0.07968639731407165
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,32767,0.14791680574417115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,32767,0.08007199764251709
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,65535,0.14801119565963744
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,65535,0.2844079971313477
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,131071,0.28826720714569093
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,131071,0.555779218673706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,1,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,3,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,16383,0.04526880085468292
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,7,0.00862400010228157
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,1,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,15,0.008843199908733368
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,15,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,31,0.008729600161314011
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,63,0.008968000113964082
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,63,0.008771199733018875
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,127,0.00883520022034645
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,127,0.008528000116348267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,255,0.008716800063848496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,255,0.009334400296211243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,511,0.008617600053548813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,511,0.008563199639320373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,1023,0.00939679965376854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,1023,0.008603200316429138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,2047,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,2047,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,4095,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,4095,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,8191,0.012516799569129943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,3,0.008635199815034866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,8191,0.012582400441169738
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,16383,0.01666560024023056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,16383,0.01656319946050644
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,32767,0.018300800025463103
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,65535,0.02014079988002777
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,65535,0.018795199692249298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,131071,0.02298399955034256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,131071,0.021009600162506102
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,7,0.00849440023303032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,1,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,3,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,3,0.008585599809885025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,7,0.010667199641466141
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,7,0.008641599863767623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,15,0.010113599896430969
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,15,0.00859839990735054
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,31,0.010552000254392624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,31,0.010204800218343735
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,63,0.010416000336408614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,63,0.010195200145244599
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,127,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,31,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,127,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,255,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,255,0.010454399883747101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,511,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,511,0.010444799810647965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,1023,0.010440000146627427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,2047,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,2047,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,1,0.008511999994516373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,4095,0.011249600350856781
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,4095,0.010556799918413162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,8191,0.01451359987258911
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,8191,0.012547199428081513
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,16383,0.01860159933567047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,16383,0.016782400012016297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,32767,0.018798400461673737
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,32767,0.0187376007437706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,65535,0.020934399962425233
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,131071,0.03931680023670196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,131071,0.02266079932451248
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,1,0.009665600210428237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,1,0.008566399663686752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,3,0.0095040000975132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,3,0.00852160006761551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,7,0.008587200194597244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,7,0.008516799658536911
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,32767,0.01658719927072525
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,15,0.008507200330495835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,15,0.008395200222730636
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,31,0.010367999970912933
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,1023,0.010435199737548828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,31,0.008691199868917466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,63,0.008654399961233138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,63,0.008694399893283845
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,127,0.008604799956083297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,255,0.01003360003232956
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,255,0.008716800063848496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,511,0.010070399940013885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,511,0.008630400151014328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,1023,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,1023,0.008646400272846222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,2047,0.01067039966583252
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,2047,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,4095,0.012849600613117218
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,4095,0.010856000334024429
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,8191,0.027460798621177673
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,8191,0.014854399859905243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,127,0.00857279971241951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,16383,0.045423999428749084
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,16383,0.02879360020160675
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,32767,0.07995359897613526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,32767,0.045480000972747806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,65535,0.1480031967163086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,65535,0.0802016019821167
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,65535,0.01884479969739914
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,1,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,131071,0.14988160133361816
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,1,0.008561599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,3,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,3,0.008488000184297562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,7,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,7,0.00854720026254654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,15,0.010471999645233154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,15,0.0102463997900486
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,31,0.010608000308275222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,31,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,63,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,63,0.010545600205659866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,127,0.00889120027422905
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,127,0.01037919968366623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,255,0.008612799644470214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,511,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,511,0.008455999940633774
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,1023,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,1023,0.008449599891901017
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,2047,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,2047,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,4095,0.01250080019235611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,4095,0.010564800351858139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,8191,0.014523200690746307
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,8191,0.014227199554443359
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,16383,0.016707199811935424
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,16383,0.014668799936771393
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,32767,0.018768000602722167
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,32767,0.016689600050449373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,65535,0.03494719862937927
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,65535,0.018632000684738158
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,131071,0.05190399885177612
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,131071,0.0350735992193222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,1,0.00849440023303032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,3,0.008513599634170532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,3,0.008686400204896926
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,7,0.00912640020251274
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,7,0.008641599863767623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,15,0.008628799766302108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,15,0.008475200086832047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,31,0.008636800199747085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,31,0.008638399839401244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,63,0.009452799707651139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,255,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,63,0.008609600365161896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,127,0.008497600257396699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,127,0.008526399731636047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,255,0.008470399677753449
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,255,0.008560000360012055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,511,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,1023,0.012608000636100769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,1023,0.010699199885129929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,2047,0.0274399995803833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,2047,0.01324319988489151
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,4095,0.04439359903335571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,4095,0.027300798892974855
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,1,0.008387199789285659
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,8191,0.07841920256614685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,131071,0.28458878993988035
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,8191,0.045335999131202696
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,16383,0.08015360236167908
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,32767,0.2836575984954834
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,32767,0.14826560020446777
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,65535,0.5562399864196778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,65535,0.28646559715270997
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,511,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,3,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,3,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,7,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,7,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,15,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,15,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,31,0.010465600341558457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,16383,0.14700959920883178
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,31,0.010470400005578995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,63,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,63,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,127,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,127,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,255,0.010655999928712846
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,255,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,511,0.012683199346065521
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,511,0.0124719999730587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,1023,0.027555200457572936
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,1023,0.014713600277900696
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,2047,0.04520159959793091
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,2047,0.02783840000629425
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,4095,0.07977759838104248
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,4095,0.045705598592758176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,8191,0.14829280376434326
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,8191,0.08145599961280822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,1,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,16383,0.28704960346221925
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,1,0.012665599584579468
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,1,0.012590399384498597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,16383,0.15064799785614014
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,3,0.012511999905109405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,7,0.012588800489902496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,7,0.012486399710178375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,15,0.012510399520397186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,15,0.012489599734544754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,31,0.012548799812793731
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,31,0.012556800246238708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,63,0.012457600235939026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,127,0.012680000066757202
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,127,0.012606400251388549
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,255,0.012708799540996551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,255,0.012511999905109405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,511,0.02874079942703247
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,511,0.014918400347232819
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,1023,0.04587199985980987
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,1023,0.029028800129890443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,2047,0.08150079846382141
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,1,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,2047,0.047521600127220155
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,4095,0.15157920122146606
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,4095,0.08222560286521911
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,3,0.012465599924325943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,1,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,1,0.009670399874448777
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,8191,0.15172319412231444
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,8191,0.29160959720611573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,3,0.01005759984254837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,63,0.012615999579429627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,7,0.010391999781131745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,7,0.008574400097131729
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,15,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,15,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,31,0.010422399640083313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,31,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,63,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,63,0.008566399663686752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,127,0.01048479974269867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,255,0.01040000021457672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,255,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,511,0.010342399775981902
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,511,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,1023,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,1023,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,2047,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,4095,0.01260959953069687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,4095,0.012435200065374375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,8191,0.014574399590492249
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,8191,0.012656000256538392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,16383,0.01656640022993088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,16383,0.014705599844455719
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,32767,0.03151040077209473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,32767,0.016697600483894348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,65535,0.04944159984588623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,3,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,65535,0.032287999987602234
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,131071,0.0828000009059906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,131071,0.04958719909191132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,1,0.01666239947080612
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,3,0.016680000722408293
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,3,0.016539199650287627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,127,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,7,0.016663999855518342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,7,0.016595199704170227
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,15,0.016705599427223206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,15,0.01658879965543747
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,31,0.016651199758052827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,31,0.016595199704170227
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,63,0.016630400717258454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,2047,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,127,0.01672160029411316
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,127,0.016739200055599212
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,255,0.016663999855518342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,511,0.04533919990062714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,511,0.02887359857559204
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,1023,0.0788927972316742
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,1023,0.04694559872150421
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,2047,0.14736640453338623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,1,0.016702400147914888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,2047,0.08187519907951354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,63,0.0165583997964859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,4095,0.28403520584106445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,4095,0.14980640411376953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,1,0.027003198862075806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,3,0.027060800790786745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,3,0.024902400374412537
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,7,0.02701919972896576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,7,0.024963200092315674
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,15,0.027001601457595826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,15,0.024903999269008638
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,31,0.026958400011062623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,31,0.02496960014104843
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,63,0.027060800790786745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,63,0.025006398558616638
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,127,0.03115360140800476
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,127,0.02499839961528778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,255,0.04538719952106476
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,255,0.031108799576759338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,255,0.02680639922618866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,511,0.08022080063819885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,511,0.047654399275779726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,1023,0.08256000280380249
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,2047,0.2841120004653931
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,1,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,2047,0.1513808012008667
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,1,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,3,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,3,0.010737600177526474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,7,0.010465600341558457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,15,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,15,0.00846719965338707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,31,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,31,0.009657599776983262
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,63,0.010468800365924836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,63,0.009774400293827057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,127,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,127,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,255,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,255,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,1,0.02483679950237274
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,511,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,511,0.010435199737548828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,1023,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,1023,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,1023,0.14792959690093993
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,2047,0.0104032002389431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,4095,0.012598399817943574
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,4095,0.01231039986014366
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,8191,0.014640000462532044
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,8191,0.01250080019235611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,16383,0.030270400643348693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,32767,0.04761599898338318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,32767,0.030934399366378783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,65535,0.08225439786911011
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,7,0.010921599715948105
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,65535,0.04820320010185242
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,131071,0.08339999914169312
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,1,0.02265920042991638
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,1,0.020795199275016784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,3,0.02266719937324524
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,3,0.022708800435066224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,7,0.02290560007095337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,7,0.022801600396633148
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,15,0.022785599529743194
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,15,0.020796799659729005
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,31,0.022726400196552275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,31,0.021436800062656403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,63,0.022724799811840057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,16383,0.01462240070104599
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,63,0.02091519981622696
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,127,0.022726400196552275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,127,0.0212336003780365
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,255,0.02086720019578934
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,511,0.018742400407791137
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,511,0.018753600120544434
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,1023,0.020788800716400147
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,131071,0.15004960298538209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,1023,0.02086720019578934
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,2047,0.02282879948616028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,2047,0.024926400184631346
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,1,0.010585600137710571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,1,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,3,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,3,0.01040479987859726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,7,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,7,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,15,0.01067039966583252
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,15,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,31,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,31,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,63,0.010564800351858139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,63,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,127,0.010639999806880952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,127,0.010424000024795533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,255,0.010657600313425063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,255,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,511,0.012425599992275238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,511,0.01069599986076355
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,1023,0.014684799313545226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,1023,0.014023999869823455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,2047,0.02978079915046692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,255,0.021083199977874757
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,2047,0.018607999384403228
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,1,0.012531200051307678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,1,0.012670400738716125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,3,0.012628799676895142
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,3,0.012598399817943574
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,7,0.012675200402736665
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,15,0.013099199533462525
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,15,0.01268800050020218
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,31,0.013443200290203095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,31,0.01268640011548996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,63,0.014564800262451171
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,63,0.012787200510501862
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,127,0.014737600088119506
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,2047,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,127,0.012758399546146392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,255,0.014617599546909332
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,255,0.01260959953069687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,511,0.016784000396728515
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,511,0.014723199605941772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,1023,0.029500800371170043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,1023,0.018960000574588777
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,2047,0.048607999086380006
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,2047,0.03323040008544922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,1,0.014745600521564484
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,1,0.012779200077056884
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,3,0.014628799259662628
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,3,0.012755200266838074
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,7,0.014678399264812469
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,7,0.012596799433231354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,7,0.012646399438381195
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,15,0.012540799379348756
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,31,0.012508800625801087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,63,0.014987200498580933
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,63,0.01252799928188324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,127,0.014593599736690522
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,255,0.014563199877738953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,255,0.012585599720478059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,511,0.028880000114440918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,511,0.01661919951438904
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,1023,0.04732159972190857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,1023,0.03049280047416687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,2047,0.08258240222930908
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,2047,0.04903840124607086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,1,0.020691199600696562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,1,0.021358400583267212
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,3,0.02066880017518997
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,3,0.02069759964942932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,7,0.020776000618934632
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,7,0.020667199790477753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,15,0.014555199444293976
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,15,0.02083519995212555
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,31,0.014608000218868256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,31,0.020899200439453126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,63,0.020891200006008147
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,63,0.020891200006008147
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,127,0.020822399854660036
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,127,0.020712000131607056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,127,0.012731200456619263
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,255,0.020735999941825865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,255,0.021007999777793884
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,511,0.018620799481868743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,511,0.018958400189876556
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,1023,0.014654399454593658
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,2047,0.014750400185585022
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,2047,0.014692799746990204
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,1,0.01464959979057312
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,1,0.014567999541759491
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,3,0.014753599464893342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,3,0.014585599303245544
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,7,0.014665600657463074
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,7,0.014721600711345673
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,15,0.014726400375366211
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,31,0.02064799964427948
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,31,0.014636799693107605
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,31,0.014574399590492249
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,63,0.014537599682807923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,63,0.014617599546909332
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,127,0.016059200465679168
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,127,0.014628799259662628
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,255,0.01459999978542328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,255,0.014560000598430633
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,511,0.01448799967765808
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,511,0.01387840062379837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,1023,0.012601600587368011
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,1023,0.014988799393177033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,1023,0.012513600289821625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,2047,0.01292639970779419
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,2047,0.012503999471664428
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,1,0.011588799953460693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,3,0.012305600196123123
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,3,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,7,0.012462399899959564
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,7,0.011159999668598175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,15,0.012483199685811996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,15,0.012484800070524216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,31,0.01247519999742508
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,15,0.020763200521469117
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,31,0.011790399998426437
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,63,0.012356799840927125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,63,0.010592000186443329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,127,0.012516799569129943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,127,0.010649599879980088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,255,0.012620800733566284
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,255,0.010665600001811982
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,511,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,511,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,1023,0.010662399977445603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,1023,0.010655999928712846
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,1,0.012700800597667695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,2047,0.01255200058221817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,1,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,2047,0.012918399274349212
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,1,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,3,0.010974399745464325
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,3,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,7,0.010969600081443787
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,7,0.010707200318574906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,15,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,15,0.010996799916028976
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,31,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,31,0.010678400099277497
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,63,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,127,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,63,0.01075040027499199
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,127,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,255,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,511,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,511,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,1023,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,1023,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,2047,0.01225920021533966
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,2047,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,1,0.020703999698162077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,1,0.020798400044441223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,3,0.02080159932374954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,7,0.020659199357032774
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,7,0.020772799849510193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,15,0.02072319984436035
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,15,0.02069759964942932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,15,0.014659200608730317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,31,0.02077919989824295
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,31,0.020703999698162077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,63,0.020713600516319274
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,63,0.020785599946975708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,127,0.02080000042915344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,127,0.020686399936676026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,255,0.020729599893093108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,255,0.020576000213623047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,511,0.01852799952030182
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,511,0.016760000586509706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,255,0.010835199803113937
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,1023,0.014699199795722961
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,2047,0.01467359960079193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,2047,0.0147024005651474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,1,0.016590400040149687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,1,0.014627200365066529
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,3,0.014584000408649444
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,7,0.014672000706195832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,7,0.014689600467681885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,15,0.014686399698257446
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,15,0.014699199795722961
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,31,0.014800000190734863
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,3,0.020710399746894835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,31,0.014739200472831726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,63,0.01462559998035431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,63,0.014777599275112152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,127,0.014571200311183929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,127,0.014849600195884705
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,255,0.014774399995803832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,511,0.01459999978542328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,511,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,1023,0.012742400169372559
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,1023,0.01252480000257492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,2047,0.012756800651550293
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,1023,0.014779199659824372
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,2047,0.012567999958992004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,1,0.012510399520397186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,3,0.012495999783277511
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,3,0.012468799948692322
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,3,0.016531200706958772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,7,0.012465599924325943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,7,0.012494400143623352
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,15,0.01252640038728714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,31,0.012627199292182922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,31,0.01249760016798973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,63,0.01252640038728714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,63,0.012383999675512314
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,127,0.012479999661445617
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,127,0.012561599910259246
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,255,0.014567999541759491
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,255,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,511,0.01085280030965805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,511,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,1023,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,1023,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,2047,0.012488000094890594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,1,0.012611199915409089
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,2047,0.01271039992570877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,15,0.012439999729394913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,1,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,3,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,3,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,7,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,7,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,15,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,255,0.01255359947681427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,15,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,31,0.01064160019159317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,31,0.010556799918413162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,63,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,63,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,127,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,127,0.010609599947929382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,255,0.01080000028014183
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,255,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,511,0.010665600001811982
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,511,0.01064160019159317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,1023,0.011247999966144562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,1023,0.010652799904346467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,2047,0.012803199887275695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,2047,0.012508800625801087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,1,0.010689599812030793
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,1,0.020787200331687926
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,3,0.02282879948616028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,3,0.02078399956226349
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,7,0.022724799811840057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,7,0.020742399990558623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,15,0.022700800001621245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,15,0.02099840044975281
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,31,0.02279520034790039
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,31,0.020790399610996248
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,63,0.02282560020685196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,63,0.020817600190639496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,127,0.022710399329662324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,127,0.020923200249671935
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,255,0.021003200113773345
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,255,0.020737600326538087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,511,0.018700799345970152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,511,0.01825920045375824
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,1023,0.016577599942684172
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,1023,0.015622399747371674
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,2047,0.018612800538539885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,1,0.016735999286174773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,1,0.016568000614643096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,3,0.01666080057621002
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,7,0.01659200042486191
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,7,0.014847999811172486
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,15,0.01658080071210861
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,1,0.022753599286079406
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,15,0.014884799718856812
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,31,0.016732800006866454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,31,0.01669600009918213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,63,0.01658879965543747
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,63,0.016648000478744505
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,127,0.016808000206947327
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,127,0.016755199432373045
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,255,0.01669439971446991
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,255,0.016494399309158324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,511,0.014632000029087067
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,511,0.014577600359916686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,1023,0.01650079935789108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,1023,0.014723199605941772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,2047,0.018775999546051025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,2047,0.016700799763202667
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,1,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,1,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,3,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,7,0.01058719977736473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,7,0.010470400005578995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,2047,0.016964800655841827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,15,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,15,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,31,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,63,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,63,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,127,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,127,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,255,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,255,0.01069599986076355
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,511,0.012595200538635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,511,0.01064160019159317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,1023,0.014667199552059173
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,1023,0.012600000202655792
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,2047,0.02906079888343811
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,3,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,2047,0.016729600727558136
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,1,0.012483199685811996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,3,0.012432000041007996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,3,0.010595200210809707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,7,0.012489599734544754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,31,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,15,0.01218239963054657
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,15,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,31,0.012451200187206269
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,31,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,63,0.012411200255155564
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,1,0.010545600205659866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,63,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,127,0.010694400221109391
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,3,0.016676799952983858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,127,0.010567999631166457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,7,0.010636799782514573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,255,0.012510399520397186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,255,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,511,0.012596799433231354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,1023,0.028601598739624024
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,1023,0.01658879965543747
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,2047,0.04572480022907257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,1,0.020820799469947814
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,1,0.0208639994263649
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,3,0.0208079993724823
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,3,0.020848000049591066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,7,0.02104319930076599
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,7,0.020734399557113647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,15,0.020814399421215057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,15,0.02083040028810501
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,31,0.021014399826526642
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,31,0.020819200575351714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,63,0.02085919976234436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,63,0.02141599953174591
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,127,0.02083680033683777
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,511,0.014542399346828461
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,255,0.020857599377632142
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,255,0.020827199518680572
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,511,0.01838880032300949
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,511,0.016833600401878358
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,1023,0.014703999459743499
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,1023,0.014678399264812469
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,2047,0.014744000136852264
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,1,0.01658399999141693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,1,0.014740799367427827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,3,0.016622400283813475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,3,0.015067200362682342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,7,0.016572800278663636
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,7,0.014766399562358857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,15,0.014684799313545226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,15,0.014734399318695069
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,31,0.014692799746990204
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,31,0.01475680023431778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,63,0.016630400717258454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,63,0.015385599434375763
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,127,0.0166143998503685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,127,0.02113119959831238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,127,0.01643519997596741
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,255,0.014569599926471711
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,511,0.012740799784660339
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,2047,0.029043200612068176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,511,0.012559999525547028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,1023,0.012656000256538392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,2047,0.014655999839305878
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,1023,0.012531200051307678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,2047,0.012622399628162384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,2047,0.012595200538635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,1,0.01250080019235611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,1,0.012600000202655792
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,3,0.012615999579429627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,3,0.0125231996178627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,7,0.012670400738716125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,7,0.012622399628162384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,15,0.012513600289821625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,15,0.012656000256538392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,31,0.012627199292182922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,63,0.012705600261688233
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,63,0.012535999715328216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,127,0.012620800733566284
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,127,0.012587200105190276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,255,0.012428800016641617
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,255,0.012531200051307678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,511,0.012515200674533844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,511,0.010606399923563003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,1023,0.01255200058221817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,1023,0.010582400113344192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,2047,0.012732799351215362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,2047,0.012572799623012543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,1,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,1,0.010542400181293488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,3,0.01048479974269867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,7,0.010380800068378448
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,3,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,7,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,15,0.010416000336408614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,31,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,31,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,255,0.014679999649524688
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,63,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,63,0.010401599854230881
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,127,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,127,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,255,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,255,0.010526400059461594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,511,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,511,0.01085439994931221
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,1023,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,1023,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,2047,0.01111840009689331
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,31,0.012558400630950928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,2047,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,1,0.012585599720478059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,1,0.012518399953842163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,3,0.012451200187206269
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,3,0.01263200044631958
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,7,0.012744000554084778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,15,0.01263359934091568
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,7,0.01255200058221817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,15,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,31,0.012753599882125854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,31,0.012603199481964112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,63,0.012651200592517852
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,127,0.012732799351215362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,15,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,127,0.012559999525547028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,255,0.012545600533485413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,511,0.014552000164985656
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,511,0.012695999443531036
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,1023,0.01740960031747818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,1023,0.016729600727558136
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,1,0.012676799297332763
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,1,0.012694400548934937
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,3,0.012705600261688233
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,3,0.012678399682044983
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,7,0.012676799297332763
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,63,0.01265919953584671
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,7,0.01255200058221817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,15,0.012731200456619263
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,31,0.012708799540996551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,31,0.012483199685811996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,255,0.012464000284671784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,63,0.012689599394798278
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,63,0.012595200538635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,127,0.01398559957742691
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,255,0.012601600587368011
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,127,0.012774400413036346
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,511,0.016702400147914888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,511,0.01485760062932968
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,1023,0.03120959997177124
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,1,0.01883520036935806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,1023,0.018939200043678283
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,1,0.018614399433135986
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,3,0.019123199582099914
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,3,0.018607999384403228
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,7,0.018990400433540344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,7,0.0186256006360054
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,15,0.01894560009241104
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,15,0.018719999492168425
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,31,0.018990400433540344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,31,0.018654400110244752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,63,0.01890240013599396
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,127,0.019100800156593323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,127,0.018321600556373597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,15,0.012600000202655792
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,255,0.01666560024023056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,511,0.03258079886436462
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,511,0.022832000255584718
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,1023,0.0514959990978241
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,255,0.01252799928188324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,1023,0.03328480124473572
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,1,0.022777600586414336
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,1,0.01871040016412735
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,3,0.022753599286079406
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,3,0.018772800266742707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,7,0.022750400006771088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,7,0.016728000342845918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,15,0.022742399573326112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,15,0.0168272003531456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,31,0.022729599475860597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,31,0.016995200514793397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,63,0.01676799952983856
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,127,0.022830399870872497
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,63,0.018751999735832213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,127,0.016771200299263
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,255,0.029016000032424927
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,255,0.017185600101947786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,511,0.04748480021953583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,511,0.02961919903755188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,1023,0.08224959969520569
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,255,0.01958719938993454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,1023,0.04850879907608032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,1,0.016579200327396394
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,1,0.014636799693107605
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,3,0.014705599844455719
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,7,0.01661120057106018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,7,0.014620800316333771
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,15,0.01666879951953888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,15,0.014723199605941772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,31,0.016596800088882445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,31,0.014803199470043183
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,63,0.01660960018634796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,63,0.014742399752140044
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,127,0.01656640022993088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,127,0.014723199605941772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,255,0.014745600521564484
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,1,0.018681600689888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,1,0.016715200245380403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,3,0.01866080015897751
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,63,0.02282560020685196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,7,0.018513600528240203
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,3,0.01679680049419403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,7,0.01671359986066818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,15,0.018462400138378143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,15,0.016756799817085267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,31,0.01684959977865219
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,255,0.01658080071210861
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,31,0.016708800196647645
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,63,0.016684800386428833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,63,0.016652800142765045
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,127,0.016785599291324615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,127,0.016648000478744505
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,255,0.01868640035390854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,255,0.016182400286197662
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,3,0.016657599806785585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,1,0.02691679894924164
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,3,0.03099679946899414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,3,0.027107200026512145
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,7,0.027113598585128785
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,15,0.031086400151252747
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,15,0.027008000016212463
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,31,0.031088000535964964
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,31,0.027009600400924684
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,63,0.030990400910377504
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,63,0.026881599426269533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,127,0.03105120062828064
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,127,0.027011200785636902
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,255,0.03478559851646423
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,255,0.028276801109313965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,1,0.02910720109939575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,1,0.03553279936313629
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,1,0.026840001344680786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,3,0.027025601267814635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,7,0.03094879984855652
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,7,0.02688319981098175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,15,0.026907199621200563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,31,0.03696959912776947
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,31,0.02693600058555603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,63,0.03714880049228668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,63,0.026976001262664796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,127,0.03842880129814148
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,127,0.02686559855937958
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,255,0.0476639986038208
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,255,0.03158720135688782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,1,0.023417599499225616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,1,0.022542400658130644
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,3,0.02476480007171631
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,3,0.022115199267864226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,7,0.02478879988193512
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,3,0.03703039884567261
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,7,0.03701600134372711
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,7,0.020788800716400147
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,15,0.037150400876998904
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,31,0.023636800050735474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,31,0.022111999988555908
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,63,0.023179200291633607
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,63,0.022111999988555908
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,127,0.021998399496078493
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,1,0.026358398795127868
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,1,0.022945599257946016
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,3,0.026872000098228453
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,3,0.022908799350261688
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,7,0.02678399980068207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,7,0.02290399968624115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,15,0.026262399554252625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,15,0.022809599339962006
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,31,0.02610880136489868
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,31,0.022920000553131103
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,63,0.026465600728988646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,63,0.02285120040178299
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,127,0.027508801221847533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,127,0.023001599311828613
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,15,0.024732799828052522
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,1,0.051939201354980466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,1,0.047353601455688475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,3,0.05175039768218994
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,3,0.04749439954757691
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,7,0.05173919796943664
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,127,0.02375520020723343
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,7,0.047367998957633974
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,15,0.04666079878807068
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,31,0.05284640192985535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,31,0.046321600675582886
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,63,0.05502399802207947
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,63,0.04640640020370483
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,127,0.058550399541854856
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,127,0.0485727995634079
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,1,0.04538399875164032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,3,0.06580640077590942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,3,0.04538399875164032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,7,0.0659775972366333
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,7,0.04538240134716034
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,15,0.06579840183258057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,15,0.021598400175571443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,15,0.045521599054336545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,31,0.0658896028995514
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,31,0.04541119933128357
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,63,0.06623839735984802
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,63,0.04598560035228729
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,15,0.05170239806175232
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,127,0.06743839979171753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,127,0.04742079973220825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,1,0.0208639994263649
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,1,0.02082560062408447
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,3,0.020916800200939178
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,3,0.02078080028295517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,7,0.020908799767494202
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,1,0.06578400135040283
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,15,0.020852799713611602
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,31,0.021801599860191347
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,31,0.020871999859809875
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,63,0.021355199813842773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,63,0.02078240066766739
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,127,0.021940800547599792
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,127,0.020849600434303284
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,255,0.02086720019578934
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,255,0.020662400126457214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,511,0.018643200397491455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,511,0.01855839937925339
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,1023,0.016523200273513793
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,1023,0.01462399959564209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,2047,0.016463999450206757
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,2047,0.014753599464893342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,1,0.014662399888038635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,7,0.020916800200939178
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,1,0.014681600034236908
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,15,0.020803199708461763
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,3,0.014627200365066529
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,7,0.014644800126552582
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,7,0.014665600657463074
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,15,0.014608000218868256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,15,0.014659200608730317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,31,0.016574400663375854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,31,0.014735999703407287
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,63,0.016630400717258454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,63,0.01462559998035431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,127,0.014727999269962311
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,127,0.016575999557971954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,255,0.016011199355125426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,255,0.014660799503326416
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,511,0.014548799395561219
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,511,0.012555199861526489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,1023,0.013062399625778199
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,1023,0.012488000094890594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,2047,0.014419199526309967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,2047,0.012545600533485413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,1,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,3,0.011539199948310852
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,3,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,7,0.011598400026559829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,7,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,15,0.011635199934244157
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,15,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,31,0.011902400106191636
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,31,0.010635200142860412
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,3,0.01462559998035431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,63,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,63,0.010625600069761276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,127,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,127,0.010567999631166457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,255,0.010552000254392624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,511,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,511,0.010419200360774993
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,1023,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,1023,0.010569600015878677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,2047,0.012583999335765839
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,2047,0.012388800084590913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,1,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,1,0.010571199655532836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,3,0.010571199655532836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,1,0.012479999661445617
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,7,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,7,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,15,0.010555200278759003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,15,0.010369600355625152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,31,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,31,0.01032319962978363
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,63,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,63,0.010388799756765366
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,127,0.010526400059461594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,127,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,255,0.01048479974269867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,255,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,255,0.010567999631166457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,511,0.010592000186443329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,511,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,1023,0.012366399914026261
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,1023,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,2047,0.012510399520397186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,2047,0.012529599666595458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,1,0.037248000502586365
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,1,0.03303999900817871
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,3,0.03711200058460236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,3,0.03299359977245331
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,7,0.037110400199890134
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,7,0.03297599852085113
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,15,0.037201601266860965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,15,0.03298879861831665
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,31,0.03764959871768951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,31,0.03306719958782196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,63,0.04003680050373078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,63,0.03299359977245331
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,1,0.039073601365089417
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,3,0.04334399998188019
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,3,0.03723039925098419
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,7,0.0433135986328125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,7,0.03760640025138855
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,15,0.04351359903812409
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,15,0.03916319906711578
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,31,0.045440000295639035
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,31,0.039323198795318606
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,63,0.04672319889068603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,63,0.03781439960002899
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,1,0.09686239957809448
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,3,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,1,0.0845359981060028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,3,0.09679679870605469
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,3,0.08428639769554139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,7,0.09811679720878601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,7,0.08453440070152282
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,15,0.10019199848175049
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,31,0.10143040418624878
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,31,0.08655040264129639
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,63,0.10296000242233276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,63,0.08889920115470887
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,1,0.12356640100479126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,1,0.08229439854621887
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,3,0.12337919473648071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,3,0.08238720297813415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,7,0.12346080541610718
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,7,0.08374559879302979
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,15,0.0844223976135254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,31,0.12539680004119874
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,31,0.08443999886512757
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,15,0.08446400165557862
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,63,0.12549920082092286
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,63,0.08429120182991028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,1,0.055848002433776855
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,3,0.06796640157699585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,3,0.056329602003097536
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,7,0.06826879978179931
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,7,0.056227201223373414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,15,0.06879680156707764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,15,0.05766400098800659
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,31,0.06878560185432434
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,15,0.12345600128173828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,31,0.05773760080337524
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,1,0.08077600002288818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,1,0.043249601125717164
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,3,0.08040480017662048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,3,0.0679744005203247
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,7,0.08179680109024048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,7,0.06788960099220276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,15,0.1279695987701416
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,15,0.06864479780197144
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,31,0.08281279802322387
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,31,0.06943519711494446
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,1,0.1887488007545471
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,1,0.16274720430374146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,3,0.1888592004776001
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,3,0.16237280368804932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,7,0.19026720523834229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,7,0.1629520058631897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,15,0.19107680320739745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,15,0.16354559659957885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,31,0.1934239983558655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,31,0.16456160545349122
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,1,0.23847520351409912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,1,0.15803680419921876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,3,0.23806400299072267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,3,0.15803040266036988
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,7,0.23848960399627686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,1,0.06789439916610718
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,15,0.23802878856658935
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,31,0.23835361003875732
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,31,0.1581936001777649
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,1,0.020972800254821778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,1,0.06851199865341187
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,1,0.020883199572563172
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,3,0.020960000157356263
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,3,0.02083519995212555
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,7,0.020812800526618956
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,7,0.020815999805927278
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,15,0.020956799387931824
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,15,0.020796799659729005
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,31,0.020871999859809875
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,63,0.021063999831676485
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,63,0.020924800634384157
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,127,0.02091519981622696
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,127,0.020875200629234314
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,255,0.02088479995727539
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,255,0.020846399664878845
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,511,0.018798400461673737
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,7,0.1581104040145874
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,1023,0.014697599411010741
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,15,0.15799520015716553
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,2047,0.01603520065546036
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,2047,0.014617599546909332
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,1,0.01627359986305237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,1,0.014577600359916686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,3,0.015572799742221833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,31,0.02078240066766739
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,3,0.016499200463294984
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,7,0.016515199840068818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,7,0.016575999557971954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,15,0.016521599888801575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,15,0.016612799465656282
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,31,0.016633599996566772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,31,0.01658560037612915
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,63,0.016550399363040924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,511,0.018606400489807128
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,63,0.016675199568271636
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,127,0.01664479970932007
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,127,0.016494399309158324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,1023,0.014526399970054626
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,255,0.01669439971446991
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,511,0.014574399590492249
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,511,0.014567999541759491
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,1023,0.012614400684833526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,1023,0.012703999876976013
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,2047,0.014635199308395385
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,1,0.012516799569129943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,2047,0.014577600359916686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,1,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,3,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,3,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,7,0.012678399682044983
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,7,0.012567999958992004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,15,0.01263200044631958
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,15,0.012535999715328216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,31,0.012561599910259246
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,31,0.012566399574279786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,63,0.012606400251388549
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,127,0.012566399574279786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,127,0.012579199671745301
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,255,0.012604799866676331
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,255,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,511,0.01080000028014183
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,511,0.010814400017261505
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,1023,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,255,0.016539199650287627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,1023,0.012507200241088867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,2047,0.014790399372577668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,1,0.009889599680900574
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,1,0.008631999790668487
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,3,0.00942239984869957
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,3,0.008671999722719193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,7,0.01061599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,7,0.009444800019264222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,15,0.008510400354862214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,15,0.009011200070381165
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,31,0.010384000092744827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,31,0.009067200124263763
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,63,0.01040479987859726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,63,0.009905599802732468
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,127,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,127,0.009838400036096573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,255,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,255,0.009679999947547913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,511,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,511,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,1023,0.012583999335765839
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,1023,0.01157120019197464
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,2047,0.027076798677444457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,2047,0.014679999649524688
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,1,0.019019199907779692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,1,0.018883199989795686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,3,0.020747199654579163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,3,0.019523200392723084
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,7,0.02075680047273636
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,7,0.019068799912929535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,15,0.020745599269866945
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,15,0.019083200395107268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,31,0.02078240066766739
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,2047,0.014635199308395385
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,31,0.019153599441051484
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,63,0.020894399285316466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,63,0.01882079988718033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,127,0.018705600500106813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,255,0.020059199631214143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,255,0.018755200505256652
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,511,0.017339199781417847
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,511,0.0165583997964859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,1023,0.018862399458885192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,1023,0.017127999663352968
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,2047,0.022947199642658234
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,2047,0.020720000565052032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,1,0.010707200318574906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,1,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,3,0.010300800204277039
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,3,0.01058719977736473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,7,0.010424000024795533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,7,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,15,0.010630399733781815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,15,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,31,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,31,0.010595200210809707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,63,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,63,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,127,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,127,0.010596799850463866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,255,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,255,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,511,0.012596799433231354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,511,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,1023,0.014655999839305878
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,1023,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,127,0.02045920044183731
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,2047,0.029105600714683533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,2047,0.016916799545288085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,1,0.012544000148773193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,1,0.012697599828243256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,3,0.012731200456619263
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,7,0.01295199990272522
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,63,0.012556800246238708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,7,0.012731200456619263
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,15,0.012870399653911591
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,15,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,31,0.012636800110340119
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,31,0.012723200023174286
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,63,0.012624000012874604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,63,0.012639999389648438
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,127,0.012936000525951386
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,127,0.022787199914455415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,255,0.012628799676895142
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,255,0.012755200266838074
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,511,0.014819200336933135
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,1023,0.030689600110054015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,1023,0.018911999464035035
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,2047,0.04875679910182953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,2047,0.033371201157569884
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,1,0.014708800613880158
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,1,0.01279519945383072
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,3,0.01263200044631958
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,3,0.015243199467658997
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,3,0.012601600587368011
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,7,0.01467519998550415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,7,0.012508800625801087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,15,0.014830400049686433
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,15,0.012729600071907043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,31,0.01462559998035431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,31,0.01255359947681427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,63,0.014801600575447082
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,63,0.012521600723266602
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,127,0.014612799882888794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,127,0.012571200728416443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,255,0.012831999361515046
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,255,0.015316799283027649
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,511,0.016769599914550782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,511,0.0166143998503685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,1023,0.047849598526954654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,2047,0.08228639960289001
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,1,0.018592000007629395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,1,0.018641600012779237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,3,0.018595199286937713
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,3,0.0186256006360054
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,7,0.018619200587272643
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,7,0.018649600446224213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,15,0.018622399866580965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,15,0.0186256006360054
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,31,0.01863519996404648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,31,0.01860000044107437
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,63,0.018646399676799773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,63,0.01863519996404648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,127,0.018592000007629395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,127,0.018648000061511995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,255,0.018624000251293182
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,255,0.0166703999042511
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,511,0.01653600037097931
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,511,0.02889919877052307
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,511,0.014556799829006196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,1023,0.012675200402736665
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,1023,0.031041601300239564
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,2047,0.014577600359916686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,2047,0.014532800018787383
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,1,0.014459200203418732
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,1,0.012622399628162384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,3,0.014511999487876893
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,3,0.01271039992570877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,7,0.012654399871826172
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,15,0.014548799395561219
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,15,0.014183999598026275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,31,0.014595200121402741
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,31,0.013502399623394012
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,63,0.01459839940071106
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,63,0.013969600200653076
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,127,0.01456640064716339
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,127,0.014528000354766845
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,255,0.01448799967765808
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,255,0.012601600587368011
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,511,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,511,0.012438400089740754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,1023,0.012544000148773193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,1023,0.012585599720478059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,1023,0.012513600289821625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,2047,0.012444800138473511
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,2047,0.011617600172758102
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,1,0.01127839982509613
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,2047,0.049579200148582456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,3,0.011051200330257416
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,7,0.012603199481964112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,3,0.010913600027561188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,7,0.014584000408649444
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,15,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,7,0.012803199887275695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,15,0.010454399883747101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,31,0.012771199643611907
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,31,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,63,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,127,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,127,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,255,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,255,0.0106175996363163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,511,0.010620799660682679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,511,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,1023,0.010603199899196624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,1023,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,2047,0.01252480000257492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,2047,0.012571200728416443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,1,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,1,0.01048479974269867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,3,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,3,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,7,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,7,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,63,0.012628799676895142
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,15,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,15,0.011052799969911575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,31,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,63,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,31,0.011043199896812439
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,63,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,127,0.011070399731397628
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,127,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,255,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,511,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,511,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,1023,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,1023,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,2047,0.012147200107574464
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,2047,0.010639999806880952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,1,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,1,0.0188511997461319
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,1,0.018731200695037843
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,3,0.01871200054883957
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,7,0.01865759938955307
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,7,0.019147199392318726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,15,0.018695999681949616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,15,0.0189423993229866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,31,0.018719999492168425
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,31,0.019091199338436126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,63,0.0189423993229866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,127,0.019006399810314177
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,127,0.019009600579738616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,255,0.010700800269842149
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,255,0.01882079988718033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,255,0.01878879964351654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,511,0.016804799437522888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,1023,0.01478399932384491
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,1023,0.014839999377727509
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,2047,0.014804799854755402
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,2047,0.014814400672912597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,1,0.014716799557209014
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,3,0.018713599443435668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,1,0.014705599844455719
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,3,0.014772799611091614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,7,0.014718399941921234
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,63,0.018692800402641298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,7,0.014336000382900237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,15,0.014545600116252898
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,15,0.014609600603580474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,31,0.014556799829006196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,31,0.014641599357128143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,63,0.014571200311183929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,127,0.014672000706195832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,127,0.01467519998550415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,511,0.016847999393939973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,255,0.014660799503326416
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,255,0.012574400007724761
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,511,0.012727999687194824
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,511,0.012455999851226807
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,1023,0.012615999579429627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,1023,0.012449599802494049
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,2047,0.012630400061607362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,2047,0.012521600723266602
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,1,0.012505599856376648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,1,0.012492799758911132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,3,0.012585599720478059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,3,0.012630400061607362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,7,0.012627199292182922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,7,0.012532800436019897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,15,0.01260959953069687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,3,0.014545600116252898
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,31,0.012656000256538392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,31,0.010552000254392624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,63,0.012476799637079239
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,63,0.010556799918413162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,127,0.011068800091743469
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,255,0.010545600205659866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,255,0.010449600219726563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,511,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,63,0.013953599333763122
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,511,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,1023,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,1023,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,2047,0.012545600533485413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,2047,0.012564800679683685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,1,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,1,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,3,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,3,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,7,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,15,0.01064160019159317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,7,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,15,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,31,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,127,0.012439999729394913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,31,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,63,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,63,0.010654400289058685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,127,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,255,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,255,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,511,0.010681600123643876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,511,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,1023,0.011191999912261963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,1023,0.010468800365924836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,2047,0.012724800407886505
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,2047,0.01244639977812767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,1,0.020694400370121
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,1,0.018671999871730804
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,3,0.019215999543666838
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,3,0.01889120042324066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,7,0.019278399646282196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,7,0.018764799833297728
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,15,0.0189423993229866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,15,0.018769599497318268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,31,0.01932159960269928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,31,0.018764799833297728
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,15,0.01053439974784851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,63,0.01868640035390854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,127,0.018751999735832213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,127,0.018643200397491455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,255,0.0186831995844841
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,127,0.010708799958229065
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,511,0.016622400283813475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,511,0.01659359931945801
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,1023,0.014748799800872802
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,2047,0.016888000071048737
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,2047,0.01658399999141693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,1,0.014740799367427827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,1,0.014638400077819825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,3,0.014791999757289887
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,3,0.014692799746990204
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,7,0.01462559998035431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,7,0.01462559998035431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,15,0.014720000326633453
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,15,0.014619199931621552
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,31,0.014912000298500061
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,31,0.014824000000953675
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,63,0.014851200580596923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,63,0.020532800257205962
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,63,0.014847999811172486
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,127,0.01467680037021637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,255,0.01467519998550415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,127,0.01483680009841919
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,255,0.014591999351978302
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,511,0.014753599464893342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,511,0.012673600018024445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,1023,0.014571200311183929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,2047,0.01780959963798523
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,2047,0.01656160056591034
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,1,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,1,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,3,0.010686399787664414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,3,0.01063840016722679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,7,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,7,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,15,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,15,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,31,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,31,0.010420800000429154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,63,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,63,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,127,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,127,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,255,0.011185599863529206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,255,0.01871519982814789
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,255,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,511,0.011528000235557556
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,1023,0.016681599617004394
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,1023,0.014537599682807923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,1023,0.013828800618648529
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,2047,0.028939199447631837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,2047,0.016788800060749055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,1,0.010675200074911118
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,1,0.010691200196743012
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,3,0.010873600095510482
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,7,0.012483199685811996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,7,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,15,0.012379200011491776
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,511,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,15,0.010542400181293488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,31,0.012196800112724305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,31,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,3,0.012196800112724305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,63,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,63,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,127,0.010675200074911118
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,127,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,255,0.010673599690198899
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,255,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,511,0.014558400213718414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,511,0.012559999525547028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,1023,0.028385600447654723
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,1023,0.014740799367427827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,2047,0.04554400146007538
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,2047,0.02961280047893524
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,1,0.018646399676799773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,1,0.018753600120544434
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,3,0.01868959963321686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,3,0.018729600310325622
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,7,0.018643200397491455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,7,0.018755200505256652
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,15,0.018644799292087556
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,15,0.018881599605083465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,31,0.020028799772262573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,31,0.018772800266742707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,63,0.018753600120544434
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,63,0.018760000169277192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,127,0.01868959963321686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,127,0.01879359930753708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,1023,0.014521600306034088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,255,0.01874080002307892
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,255,0.01807360053062439
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,511,0.016457599401474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,511,0.014611199498176575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,1023,0.014395199716091156
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,2047,0.014593599736690522
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,1,0.01462399959564209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,1,0.01279519945383072
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,3,0.014608000218868256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,7,0.014572800695896148
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,3,0.015007999539375306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,7,0.014507199823856353
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,15,0.015078400075435639
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,31,0.014604799449443817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,31,0.01459839940071106
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,63,0.014731200039386749
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,63,0.014590400457382201
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,127,0.014534400403499603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,127,0.014665600657463074
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,255,0.014575999975204468
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,255,0.014616000652313232
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,511,0.012782399356365205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,511,0.0127920001745224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,1023,0.014670400321483612
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,2047,0.01449120044708252
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,1023,0.012598399817943574
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,2047,0.012476799637079239
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,2047,0.01252480000257492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,1,0.01244639977812767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,1,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,3,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,7,0.010592000186443329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,7,0.010633599758148194
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,15,0.010769599676132202
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,15,0.014638400077819825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,15,0.010689599812030793
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,31,0.01250080019235611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,31,0.012107200175523757
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,63,0.0125231996178627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,127,0.012535999715328216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,127,0.012567999958992004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,255,0.012406399846076966
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,255,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,511,0.01064160019159317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,511,0.010592000186443329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,1023,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,1023,0.010609599947929382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,2047,0.012531200051307678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,1023,0.012697599828243256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,2047,0.012451200187206269
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,1,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,3,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,3,0.012412799894809723
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,3,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,7,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,7,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,15,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,31,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,31,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,63,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,63,0.012489599734544754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,63,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,127,0.010363200306892395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,127,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,255,0.010420800000429154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,255,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,511,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,1023,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,1,0.010892800241708755
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,1023,0.010419200360774993
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,15,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,2047,0.01249760016798973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,2047,0.010625600069761276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,1,0.012630400061607362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,3,0.01266240030527115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,3,0.012639999389648438
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,7,0.012652799487113953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,7,0.012540799379348756
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,511,0.00840959995985031
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,15,0.012671999633312225
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,15,0.012601600587368011
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,31,0.01249760016798973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,31,0.012590399384498597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,63,0.012596799433231354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,63,0.012561599910259246
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,127,0.012583999335765839
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,127,0.012564800679683685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,255,0.013424000144004822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,255,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,511,0.014596800506114959
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,511,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,1023,0.016764800250530242
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,1023,0.018731200695037843
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,1,0.012585599720478059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,1,0.012561599910259246
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,3,0.01266240030527115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,3,0.012608000636100769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,7,0.012614400684833526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,15,0.012689599394798278
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,15,0.01250080019235611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,31,0.01268800050020218
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,31,0.01255040019750595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,63,0.012569600343704223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,127,0.012572799623012543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,127,0.012481600046157837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,255,0.012548799812793731
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,255,0.01207519993185997
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,511,0.016624000668525696
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,511,0.014524799585342408
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,1023,0.030348798632621764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,1023,0.0186271995306015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,1,0.01866080015897751
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,1,0.016740800440311433
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,3,0.018787199258804323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,3,0.017015999555587767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,7,0.018641600012779237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,1,0.01239520013332367
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,15,0.018739199638366698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,7,0.016596800088882445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,15,0.016780799627304076
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,31,0.018644799292087556
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,63,0.018691200017929076
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,63,0.01669279932975769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,63,0.012307199835777282
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,127,0.018700799345970152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,127,0.01674720048904419
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,255,0.018671999871730804
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,255,0.016633599996566772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,511,0.03175680041313171
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,511,0.022815999388694764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,1023,0.05092639923095703
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,1023,0.0332399994134903
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,1,0.018352000415325163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,3,0.022777600586414336
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,3,0.016756799817085267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,7,0.022758400440216063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,7,0.016646400094032288
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,15,0.022862400114536285
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,15,0.016700799763202667
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,7,0.012598399817943574
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,31,0.01866399943828583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,63,0.022761599719524385
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,63,0.018718400597572328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,127,0.022862400114536285
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,127,0.018636800348758698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,255,0.02898240089416504
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,255,0.018628799915313722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,511,0.04742720127105713
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,1,0.022750400006771088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,511,0.029862400889396668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,1023,0.08213120102882385
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,1023,0.04876160025596619
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,1,0.01653439998626709
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,3,0.016574400663375854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,1,0.014726400375366211
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,3,0.014535999298095703
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,7,0.016603200137615202
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,7,0.014553600549697876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,15,0.016627199947834015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,31,0.016599999368190767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,31,0.014582400023937226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,31,0.022703999280929567
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,63,0.016732800006866454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,63,0.014535999298095703
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,127,0.016625599563121797
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,127,0.014537599682807923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,255,0.015881599485874177
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,31,0.01669600009918213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,255,0.01462559998035431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,1,0.017422400414943695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,1,0.016495999693870545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,3,0.01860959976911545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,3,0.015855999290943147
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,7,0.018668800592422485
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,7,0.016305600106716157
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,15,0.018643200397491455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,15,0.014679999649524688
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,31,0.018505600094795228
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,31,0.014627200365066529
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,63,0.016836799681186676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,63,0.014662399888038635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,127,0.01796479970216751
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,127,0.014723199605941772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,255,0.01733759939670563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,255,0.01462399959564209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,1,0.029558399319648744
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,1,0.026932799816131593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,3,0.03103039860725403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,3,0.026948800683021544
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,7,0.030913600325584413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,7,0.02693600058555603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,15,0.014561599493026734
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,15,0.03091999888420105
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,15,0.02686080038547516
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,31,0.028881600499153136
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,63,0.03048959970474243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,127,0.031083199381828307
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,63,0.026894399523735048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,127,0.027039998769760133
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,255,0.03312479853630066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,255,0.02688480019569397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,1,0.026948800683021544
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,3,0.03713279962539673
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,3,0.026950401067733765
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,7,0.03706879913806915
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,7,0.026918399333953857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,15,0.037108799815177916
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,15,0.026907199621200563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,31,0.037089601159095764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,31,0.02688640058040619
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,63,0.037227201461791995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,63,0.026836800575256347
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,127,0.03837119936943054
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,127,0.0269679993391037
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,255,0.04740320146083832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,255,0.031172800064086913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,1,0.022771200537681578
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,1,0.037171199917793274
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,1,0.020747199654579163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,3,0.022830399870872497
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,3,0.020759999752044678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,7,0.022720000147819518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,7,0.020788800716400147
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,15,0.022830399870872497
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,15,0.020873600244522096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,31,0.022742399573326112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,31,0.02074880003929138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,63,0.020718400180339814
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,127,0.022787199914455415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,127,0.020721599459648132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,1,0.025123199820518492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,1,0.022720000147819518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,3,0.0254287987947464
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,3,0.02274080067873001
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,7,0.024849599599838255
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,7,0.022711999714374542
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,15,0.02495039999485016
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,31,0.02699199914932251
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,15,0.022787199914455415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,31,0.026824000477790832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,31,0.022759999334812164
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,63,0.026851201057434083
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,63,0.022804799675941467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,127,0.026940798759460448
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,127,0.022728000581264497
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,1,0.051560002565383914
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,1,0.045393601059913635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,3,0.05149279832839966
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,3,0.04531840085983276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,7,0.05154079794883728
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,7,0.04543519914150238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,15,0.05156800150871277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,15,0.04540480077266693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,63,0.023219199478626253
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,31,0.05166879892349243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,31,0.04545120000839233
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,63,0.053723198175430295
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,63,0.047198399901390076
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,127,0.05642719864845276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,1,0.06597440242767334
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,1,0.04527519941329956
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,3,0.06609759926795959
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,3,0.04532800018787384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,7,0.06592320203781128
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,7,0.04531840085983276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,15,0.06579359769821166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,31,0.0678384006023407
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,31,0.04538399875164032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,63,0.06794559955596924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,63,0.04541279971599579
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,127,0.06829760074615479
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,127,0.04758400022983551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,1,0.018700799345970152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,1,0.018590399622917177
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,3,0.018828800320625304
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,3,0.01870719939470291
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,7,0.018769599497318268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,7,0.018787199258804323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,15,0.018855999410152435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,127,0.046377599239349365
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,15,0.018777599930763243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,31,0.018771199882030486
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,63,0.01879359930753708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,15,0.04532960057258606
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,127,0.01876640021800995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,127,0.018718400597572328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,255,0.01866399943828583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,255,0.016686399281024934
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,511,0.016672000288963318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,511,0.014646400511264802
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,1023,0.01459999978542328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,1023,0.014612799882888794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,2047,0.014707200229167938
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,2047,0.01459839940071106
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,1,0.01454399973154068
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,1,0.014561599493026734
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,3,0.014664000272750855
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,3,0.01456640064716339
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,31,0.01870719939470291
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,7,0.014571200311183929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,7,0.014569599926471711
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,15,0.01454399973154068
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,15,0.014561599493026734
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,31,0.014572800695896148
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,31,0.014553600549697876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,63,0.014595200121402741
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,63,0.014591999351978302
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,127,0.014590400457382201
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,127,0.014558400213718414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,255,0.014683200418949128
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,255,0.014542399346828461
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,511,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,511,0.012510399520397186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,1023,0.01252480000257492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,1023,0.0127920001745224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,1,0.01096000000834465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,1,0.010931199789047242
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,3,0.010873600095510482
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,3,0.010923200100660325
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,7,0.011057599633932113
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,7,0.010916800051927567
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,15,0.011321599781513213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,15,0.010913600027561188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,31,0.012108799815177918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,31,0.012177599966526032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,63,0.011376000195741653
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,63,0.018587200343608855
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,63,0.010737600177526474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,127,0.010585600137710571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,127,0.0108255997300148
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,255,0.010660800337791442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,511,0.010619200021028518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,255,0.010793600231409073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,2047,0.012598399817943574
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,511,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,1023,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,2047,0.012540799379348756
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,2047,0.011751999706029892
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,1,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,1,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,3,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,3,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,7,0.010655999928712846
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,15,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,15,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,31,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,31,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,63,0.010660800337791442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,63,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,127,0.01066880002617836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,127,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,255,0.010569600015878677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,255,0.010635200142860412
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,511,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,511,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,1023,0.012646399438381195
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,1023,0.01063840016722679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,2047,0.013051199913024902
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,2047,0.012596799433231354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,2047,0.012510399520397186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,1,0.03704639971256256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,1,0.03108479976654053
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,3,0.037006399035453795
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,3,0.030990400910377504
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,7,0.03693920075893402
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,7,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,15,0.03690559864044189
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,15,0.03094559907913208
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,31,0.03707999885082245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,31,0.03123520016670227
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,63,0.03720960021018982
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,63,0.0315311998128891
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,1,0.0434112012386322
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,1,0.03732320070266724
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,3,0.04340159893035889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,3,0.037273600697517395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,7,0.04332480132579804
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,7,0.03719359934329987
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,15,0.04357759952545166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,1023,0.010876800119876861
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,15,0.03724640011787415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,31,0.044200000166893
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,7,0.030955201387405394
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,63,0.04575519859790802
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,63,0.03747679889202118
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,1,0.0846015989780426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,3,0.09699519872665405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,3,0.08478879928588867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,7,0.09692959785461426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,7,0.08462079763412475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,15,0.09739519953727722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,15,0.08460639715194702
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,31,0.037252798676490784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,31,0.09902080297470092
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,63,0.100547194480896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,63,0.08532320261001587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,1,0.12522079944610595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,1,0.09652479887008666
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,1,0.08218880295753479
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,3,0.1251855969429016
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,3,0.08224480152130127
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,7,0.12491199970245362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,7,0.0823311984539032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,15,0.1255728006362915
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,31,0.1265071988105774
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,31,0.08393920063972474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,63,0.12654080390930175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,63,0.08426399827003479
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,1,0.06399199962615967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,1,0.05362240076065063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,3,0.06408320069313049
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,3,0.05370079874992371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,7,0.06410719752311707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,7,0.053907197713851926
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,15,0.06392160058021545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,31,0.06472480297088623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,15,0.05355839729309082
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,31,0.05368480086326599
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,1,0.07726240158081055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,1,0.0659824013710022
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,3,0.07834720015525817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,3,0.06605600118637085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,7,0.07882720232009888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,31,0.08421279788017273
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,15,0.08000320196151733
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,15,0.06581439971923828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,31,0.08029760122299194
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,1,0.18608479499816893
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,15,0.0821615993976593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,1,0.16115520000457764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,3,0.16053919792175292
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,7,0.18688000440597535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,7,0.16064159870147704
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,15,0.1873088002204895
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,15,0.1615072011947632
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,7,0.06589120030403137
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,31,0.18767039775848388
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,31,0.1628383994102478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,31,0.06618720293045044
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,1,0.2418384075164795
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,1,0.15784640312194825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,3,0.1560976028442383
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,3,0.24025280475616456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,7,0.24306879043579102
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,7,0.1575103998184204
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,15,0.2403791904449463
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,3,0.185207998752594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,15,0.15817439556121826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,1,0.018636800348758698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,1,0.01871200054883957
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,31,0.15689760446548462
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,3,0.018667200207710268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,3,0.01871519982814789
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,7,0.018648000061511995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,7,0.018775999546051025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,15,0.01866399943828583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,31,0.018648000061511995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,31,0.01865279972553253
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,63,0.018662400543689728
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,63,0.018796800076961516
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,127,0.018713599443435668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,127,0.018700799345970152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,255,0.01863359957933426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,255,0.01852799952030182
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,511,0.01671680063009262
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,511,0.01626240015029907
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,1023,0.01451359987258911
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,1023,0.014448000490665436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,2047,0.014864000678062438
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,2047,0.014577600359916686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,1,0.014577600359916686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,1,0.014563199877738953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,3,0.014577600359916686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,3,0.01454399973154068
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,7,0.014609600603580474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,7,0.01462399959564209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,15,0.01451520025730133
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,15,0.01456640064716339
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,31,0.014555199444293976
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,31,0.014511999487876893
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,31,0.24201440811157227
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,63,0.014567999541759491
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,127,0.014611199498176575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,15,0.018787199258804323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,127,0.014561599493026734
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,255,0.014632000029087067
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,511,0.012563200294971466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,511,0.01252480000257492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,1023,0.012488000094890594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,1023,0.012508800625801087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,2047,0.014550399780273438
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,2047,0.013390399515628815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,1,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,1,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,3,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,3,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,7,0.0124719999730587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,7,0.010769599676132202
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,15,0.012494400143623352
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,15,0.011062400043010711
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,31,0.01249760016798973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,31,0.011537600308656693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,63,0.012510399520397186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,63,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,127,0.011769600212574005
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,127,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,63,0.014747199416160584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,255,0.010596799850463866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,255,0.012383999675512314
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,255,0.01448799967765808
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,1023,0.012532800436019897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,1023,0.012345600128173827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,2047,0.014612799882888794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,2047,0.014443199336528777
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,1,0.009780800342559815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,1,0.008532799780368805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,3,0.009596800059080124
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,3,0.008539199829101562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,7,0.010382399708032609
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,7,0.008769600093364716
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,15,0.010305599868297577
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,15,0.008603200316429138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,31,0.009670399874448777
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,31,0.008535999804735184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,63,0.009838400036096573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,63,0.00915839970111847
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,127,0.010329599678516387
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,127,0.009326399862766266
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,255,0.010356800258159637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,255,0.009214399755001068
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,511,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,511,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,511,0.010424000024795533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,1023,0.012486399710178375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,511,0.010639999806880952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,1023,0.01247519999742508
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,2047,0.026966398954391478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,1,0.01659200042486191
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,1,0.016518400609493257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,3,0.014796799421310425
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,7,0.016575999557971954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,7,0.014720000326633453
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,15,0.016620799899101257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,15,0.01462399959564209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,31,0.01656640022993088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,31,0.014628799259662628
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,63,0.016547200083732606
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,63,0.016574400663375854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,127,0.016604800522327424
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,127,0.016568000614643096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,255,0.016624000668525696
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,255,0.015849600732326507
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,511,0.014528000354766845
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,511,0.014481599628925323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,1023,0.014635199308395385
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,1023,0.01653439998626709
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,2047,0.018670399487018586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,2047,0.016606399416923524
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,4095,0.03309119939804077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,4095,0.020777599513530733
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,1,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,3,0.010422399640083313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,2047,0.014619199931621552
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,3,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,3,0.016595199704170227
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,7,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,15,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,31,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,1,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,31,0.010419200360774993
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,63,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,7,0.010454399883747101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,63,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,127,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,127,0.010420800000429154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,255,0.010556799918413162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,255,0.010468800365924836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,511,0.0125231996178627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,511,0.011054400354623795
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,1023,0.014614400267601014
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,1023,0.012595200538635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,2047,0.028937599062919615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,2047,0.016673600673675536
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,4095,0.047121599316596985
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,4095,0.029971200227737426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,1,0.012595200538635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,1,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,3,0.012465599924325943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,3,0.010440000146627427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,7,0.01255359947681427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,7,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,15,0.012492799758911132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,15,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,31,0.012465599924325943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,31,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,63,0.011776000261306763
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,63,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,127,0.012345600128173827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,127,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,255,0.011497599631547928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,255,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,511,0.014532800018787383
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,511,0.012542399764060973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,1023,0.02892960011959076
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,1023,0.016513599455356597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,2047,0.04593920111656189
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,2047,0.029206401109695433
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,4095,0.0803871989250183
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,4095,0.04731520116329193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,1,0.012678399682044983
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,1,0.01250240057706833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,3,0.012532800436019897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,3,0.012491200119256973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,7,0.012483199685811996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,15,0.012515200674533844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,15,0.011899200081825257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,31,0.012443199753761292
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,63,0.012478400021791458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,63,0.01228799968957901
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,127,0.012558400630950928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,127,0.012520000338554382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,255,0.01247360035777092
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,255,0.01249760016798973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,511,0.02892000079154968
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,511,0.014561599493026734
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,1023,0.04679999947547912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,1023,0.029020801186561584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,2047,0.08203359842300414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,2047,0.048019200563430786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,15,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,4095,0.15290080308914183
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,7,0.012542399764060973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,4095,0.085590398311615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,1,0.014567999541759491
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,3,0.014612799882888794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,3,0.014521600306034088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,7,0.01459839940071106
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,7,0.014638400077819825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,15,0.014614400267601014
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,31,0.012913599610328674
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,31,0.014711999893188476
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,31,0.014347200095653535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,63,0.0147024005651474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,63,0.013665600121021271
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,127,0.013976000249385834
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,255,0.014532800018787383
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,255,0.01462399959564209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,511,0.012558400630950928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,511,0.012488000094890594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,1023,0.012486399710178375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,1023,0.010585600137710571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,2047,0.01249919980764389
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,2047,0.012428800016641617
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,4095,0.012516799569129943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,4095,0.012464000284671784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,1,0.01063840016722679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,1,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,1,0.014617599546909332
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,3,0.010684800148010255
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,3,0.012467200309038163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,7,0.010673599690198899
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,7,0.012452799826860428
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,15,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,15,0.012454400211572647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,31,0.010632000118494033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,31,0.01244639977812767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,63,0.01249919980764389
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,63,0.012503999471664428
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,127,0.012489599734544754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,127,0.014564800262451171
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,255,0.012417600303888322
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,255,0.012542399764060973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,511,0.012476799637079239
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,511,0.012511999905109405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,1023,0.012531200051307678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,1023,0.011604800075292587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,2047,0.012595200538635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,2047,0.012539200484752655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,4095,0.012567999958992004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,4095,0.01263359934091568
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,1,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,1,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,3,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,15,0.014657600224018097
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,3,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,7,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,7,0.010401599854230881
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,15,0.010628800094127654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,15,0.01066880002617836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,127,0.01257600039243698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,31,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,63,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,63,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,127,0.010526400059461594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,127,0.010654400289058685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,255,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,511,0.010652799904346467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,511,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,1023,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,1023,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,2047,0.010943999886512757
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,2047,0.010735999792814255
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,4095,0.012495999783277511
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,4095,0.012478400021791458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,1,0.010582400113344192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,1,0.008771199733018875
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,3,0.00854720026254654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,7,0.008539199829101562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,7,0.00854720026254654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,15,0.008670400083065032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,31,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,15,0.008590400218963623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,31,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,31,0.008499199897050858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,63,0.0104032002389431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,127,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,63,0.008585599809885025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,127,0.010375999659299851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,255,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,255,0.00873439982533455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,255,0.010395199805498124
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,511,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,1023,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,1023,0.010337600111961364
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,2047,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,2047,0.010628800094127654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,4095,0.012484800070524216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,1,0.014574399590492249
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,1,0.014603200554847717
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,3,0.014619199931621552
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,3,0.01456640064716339
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,7,0.014640000462532044
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,7,0.014449599385261535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,15,0.01475680023431778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,15,0.014571200311183929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,31,0.014678399264812469
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,31,0.014604799449443817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,3,0.00899680033326149
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,63,0.014660799503326416
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,63,0.01459839940071106
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,127,0.014603200554847717
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,127,0.01467359960079193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,255,0.014575999975204468
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,255,0.014526399970054626
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,511,0.01265919953584671
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,511,0.01250240057706833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,1023,0.012598399817943574
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,1023,0.012484800070524216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,2047,0.012670400738716125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,2047,0.01247519999742508
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,4095,0.01438560038805008
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,4095,0.012460800260305405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,1,0.012465599924325943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,1,0.012464000284671784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,3,0.012433599680662155
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,3,0.0124719999730587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,511,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,7,0.012468799948692322
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,15,0.012468799948692322
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,4095,0.012380799651145935
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,31,0.012559999525547028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,31,0.012478400021791458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,63,0.012539200484752655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,63,0.012513600289821625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,127,0.012580800056457519
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,127,0.012465599924325943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,255,0.012539200484752655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,255,0.012571200728416443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,511,0.012495999783277511
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,511,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,1023,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,1023,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,2047,0.01252480000257492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,2047,0.010675200074911118
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,4095,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,4095,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,1,0.01085439994931221
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,1,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,3,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,7,0.012990400195121765
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,7,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,15,0.012505599856376648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,7,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,15,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,15,0.010377600044012069
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,31,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,31,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,63,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,127,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,127,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,255,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,255,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,511,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,511,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,1023,0.010471999645233154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,1023,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,2047,0.012587200105190276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,2047,0.012382400035858155
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,4095,0.012668800354003907
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,4095,0.012467200309038163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,3,0.010632000118494033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,1,0.010406400263309478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,1,0.009788800030946732
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,3,0.010416000336408614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,3,0.010393600165843963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,7,0.011327999830245971
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,15,0.010385599732398988
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,15,0.008582399785518646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,63,0.010424000024795533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,31,0.008724799752235413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,63,0.010396800190210342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,63,0.008604799956083297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,127,0.008687999844551087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,255,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,255,0.009640000015497207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,511,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,511,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,1023,0.010545600205659866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,1023,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,2047,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,2047,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,4095,0.012606400251388549
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,4095,0.011478400230407715
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,1,0.01653439998626709
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,1,0.014707200229167938
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,3,0.016564799845218657
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,7,0.008513599634170532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,3,0.015607999265193939
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,31,0.01040000021457672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,7,0.014683200418949128
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,15,0.016467200219631196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,15,0.014641599357128143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,31,0.016547200083732606
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,63,0.01658080071210861
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,63,0.016471999883651733
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,127,0.016553600132465363
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,127,0.01650079935789108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,255,0.01650719940662384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,255,0.015113599598407745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,511,0.01454399973154068
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,511,0.014497600495815277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,1023,0.014467200636863709
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,1023,0.012596799433231354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,2047,0.014713600277900696
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,2047,0.01451680064201355
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,4095,0.01870400011539459
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,4095,0.016582399606704712
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,1,0.012383999675512314
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,1,0.012455999851226807
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,3,0.01247360035777092
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,7,0.016595199704170227
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,3,0.012558400630950928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,7,0.012452799826860428
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,7,0.012486399710178375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,15,0.012455999851226807
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,15,0.012455999851226807
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,31,0.012465599924325943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,31,0.01247360035777092
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,63,0.012470400333404541
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,127,0.012491200119256973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,63,0.012459199875593185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,127,0.012569600343704223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,255,0.012537600100040435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,255,0.012483199685811996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,511,0.011723200231790543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,511,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,1023,0.012457600235939026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,2047,0.01451839953660965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,127,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,31,0.016443200409412384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,2047,0.014535999298095703
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,4095,0.02900159955024719
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,1,0.010619200021028518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,1023,0.012732799351215362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,3,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,3,0.008558399975299835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,7,0.009935999661684037
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,7,0.008795200288295746
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,15,0.010372799634933472
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,15,0.009217599779367447
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,31,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,31,0.009723199903964997
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,63,0.010630399733781815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,63,0.009385599941015243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,127,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,127,0.009148799628019334
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,255,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,4095,0.018777599930763243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,255,0.008620800077915191
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,511,0.010659199953079224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,511,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,1023,0.012606400251388549
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,1,0.008504000306129456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,2047,0.027723199129104613
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,1023,0.010931199789047242
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,4095,0.02850399911403656
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,1,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,1,0.010334400087594986
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,3,0.0106175996363163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,7,0.010684800148010255
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,7,0.010603199899196624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,15,0.010664000362157821
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,15,0.00902559980750084
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,31,0.010692799836397171
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,31,0.00902400016784668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,63,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,63,0.009636799991130828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,127,0.010606399923563003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,127,0.010212799906730652
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,255,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,255,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,511,0.012702399492263794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,511,0.010657600313425063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,2047,0.014511999487876893
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,4095,0.046009600162506104
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,1023,0.014526399970054626
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,2047,0.0454255998134613
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,2047,0.028798401355743408
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,4095,0.08014879822731018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,1,0.014692799746990204
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,1,0.01465120017528534
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,3,0.014664000272750855
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,3,0.014707200229167938
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,7,0.014640000462532044
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,7,0.014633600413799287
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,15,0.014657600224018097
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,15,0.014945599436759948
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,31,0.014686399698257446
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,31,0.014667199552059173
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,63,0.014662399888038635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,63,0.01467680037021637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,127,0.014659200608730317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,127,0.014590400457382201
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,1023,0.027518400549888612
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,255,0.01467680037021637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,255,0.01467359960079193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,511,0.012671999633312225
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,3,0.010332799702882766
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,511,0.012627199292182922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,1023,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,1023,0.01241919994354248
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,2047,0.01255359947681427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,4095,0.04723840057849884
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,2047,0.012569600343704223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,4095,0.012579199671745301
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,1,0.012574400007724761
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,3,0.012558400630950928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,3,0.012707200646400452
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,7,0.012534399330615998
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,7,0.012558400630950928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,15,0.01265919953584671
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,31,0.012537600100040435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,31,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,63,0.012492799758911132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,63,0.012559999525547028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,127,0.012577599287033081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,127,0.012567999958992004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,255,0.01249919980764389
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,255,0.011655999720096588
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,511,0.011575999855995178
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,511,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,1023,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,1023,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,2047,0.012563200294971466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,2047,0.012649600207805634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,4095,0.014616000652313232
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,4095,0.012644800543785095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,4095,0.014502400159835815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,1,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,1,0.0125231996178627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,3,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,1,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,3,0.01040479987859726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,7,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,7,0.010571199655532836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,15,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,15,0.012590399384498597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,31,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,31,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,63,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,63,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,127,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,255,0.010424000024795533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,255,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,511,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,511,0.010734400153160096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,1023,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,1023,0.010555200278759003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,2047,0.012695999443531036
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,2047,0.010606399923563003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,4095,0.013246400654315949
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,4095,0.012614400684833526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,1,0.008566399663686752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,1,0.00859680026769638
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,15,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,3,0.0088128000497818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,3,0.008544000238180161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,7,0.00857120007276535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,7,0.008550400286912918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,15,0.008532799780368805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,15,0.008564800024032593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,31,0.008555199950933456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,127,0.010603199899196624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,31,0.008673600107431411
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,63,0.00857120007276535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,63,0.008491200208663941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,127,0.008587200194597244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,255,0.008595199882984161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,255,0.008484800159931184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,511,0.008579199761152267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,1023,0.009513600170612336
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,1023,0.008574400097131729
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,2047,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,2047,0.010999999940395355
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,4095,0.012596799433231354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,4095,0.012726399302482604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,1,0.010571199655532836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,1,0.010892800241708755
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,3,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,3,0.010660800337791442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,7,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,7,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,15,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,15,0.010739199817180634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,31,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,31,0.010686399787664414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,63,0.010768000036478043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,127,0.010819199681282043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,127,0.010652799904346467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,255,0.010785599797964096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,127,0.008524800091981888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,255,0.010649599879980088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,511,0.012571200728416443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,511,0.010760000348091126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,1023,0.014732800424098969
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,511,0.008532799780368805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,2047,0.029532799124717714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,2047,0.018475200235843658
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,1,0.014499199390411378
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,3,0.014640000462532044
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,3,0.012695999443531036
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,7,0.014641599357128143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,7,0.012744000554084778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,15,0.014596800506114959
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,15,0.012620800733566284
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,31,0.013180799782276154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,31,0.012615999579429627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,63,0.013020800054073333
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,63,0.012727999687194824
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,63,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,127,0.013382400572299957
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,255,0.014105600118637086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,255,0.01271200031042099
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,511,0.016756799817085267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,511,0.015719999372959138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,1023,0.03030720055103302
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,1023,0.02072480022907257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,2047,0.048785600066185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,2047,0.033020800352096556
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,1,0.014703999459743499
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,1,0.014494399726390838
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,1,0.012638400495052337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,3,0.014691199362277984
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,3,0.012571200728416443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,7,0.014667199552059173
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,7,0.012622399628162384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,15,0.014692799746990204
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,15,0.012563200294971466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,31,0.014814400672912597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,31,0.01446399986743927
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,63,0.01462240070104599
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,63,0.01448799967765808
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,127,0.014638400077819825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,127,0.012580800056457519
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,127,0.012601600587368011
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,255,0.012768000364303589
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,511,0.02908799946308136
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,511,0.016806399822235106
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,1023,0.04757919907569885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,1023,0.013984000682830811
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,1023,0.03105599880218506
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,2047,0.08398079872131348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,2047,0.049667200446128844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,1,0.014828799664974213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,3,0.01687040030956268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,3,0.014955200254917145
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,7,0.016820800304412842
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,7,0.01478399932384491
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,15,0.01679839938879013
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,255,0.014708800613880158
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,15,0.014668799936771393
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,31,0.01677280068397522
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,31,0.014793600142002105
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,63,0.01675039976835251
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,63,0.014923200011253357
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,127,0.016755199432373045
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,255,0.028248000144958495
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,255,0.01539520025253296
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,511,0.045470398664474485
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,511,0.02921920120716095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,1023,0.08111039996147155
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,1023,0.0478767991065979
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,1,0.01725119948387146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,1,0.012652799487113953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,2047,0.1504304051399231
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,2047,0.08415039777755737
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,1,0.01242239996790886
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,3,0.012641599774360657
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,7,0.012574400007724761
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,3,0.012670400738716125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,7,0.012558400630950928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,15,0.012579199671745301
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,15,0.012591999769210816
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,31,0.012356799840927125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,63,0.01276639997959137
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,127,0.012732799351215362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,127,0.012593600153923034
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,255,0.012649600207805634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,255,0.012627199292182922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,511,0.016732800006866454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,511,0.01467519998550415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,127,0.014672000706195832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,1,0.016766400635242464
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,3,0.018831999599933626
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,3,0.018441599607467652
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,7,0.018822400271892546
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,7,0.018287999927997588
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,15,0.018729600310325622
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,15,0.018320000171661376
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,31,0.018771199882030486
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,31,0.013648000359535218
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,31,0.01796479970216751
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,63,0.019572800397872923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,63,0.012572799623012543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,63,0.017668800055980684
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,127,0.019246399402618408
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,127,0.018129600584506987
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,255,0.018801599740982056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,255,0.018276800215244294
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,511,0.0329120010137558
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,511,0.021784000098705292
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,1,0.022787199914455415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,1,0.018611200153827667
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,3,0.018585599958896637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,7,0.02274720072746277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,7,0.01852799952030182
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,15,0.022779199481010436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,15,0.01865919977426529
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,31,0.022703999280929567
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,31,0.01858399957418442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,63,0.02271360009908676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,63,0.018559999763965607
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,127,0.022724799811840057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,127,0.018592000007629395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,1,0.018807999789714813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,255,0.029193601012229918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,511,0.047388800978660585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,1,0.027014398574829103
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,1,0.022862400114536285
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,3,0.027004799246788024
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,3,0.022940799593925476
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,7,0.026958400011062623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,7,0.022912000119686127
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,15,0.026955199241638184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,15,0.022852799296379088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,31,0.02688480019569397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,31,0.02298399955034256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,63,0.02696160078048706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,3,0.022286400198936462
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,63,0.022961600124835967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,127,0.03134720027446747
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,127,0.024801599979400634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,255,0.018675200641155243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,255,0.04750080108642578
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,511,0.030659198760986328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,255,0.03110080063343048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,1,0.01865759938955307
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,511,0.04755519926548004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,1,0.016601599752902985
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,3,0.018662400543689728
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,3,0.016612799465656282
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,7,0.01841440051794052
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,7,0.016601599752902985
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,15,0.018806399405002595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,15,0.0166143998503685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,31,0.01844799965620041
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,31,0.01656319946050644
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,63,0.01857919991016388
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,63,0.016654400527477263
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,127,0.0186256006360054
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,127,0.016732800006866454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,255,0.01658560037612915
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,1,0.030956798791885377
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,1,0.028774398565292358
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,3,0.031044799089431762
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,3,0.028638398647308348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,7,0.0309935986995697
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,7,0.02746559977531433
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,15,0.03091840147972107
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,15,0.027184000611305235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,31,0.030988800525665283
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,31,0.027268800139427184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,63,0.030958399176597595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,63,0.027057600021362305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,127,0.02717120051383972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,127,0.03135519921779632
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,255,0.03530719876289368
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,255,0.02918879985809326
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,1,0.03710080087184906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,3,0.03702079951763153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,255,0.01870879977941513
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,3,0.02690880000591278
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,7,0.037118399143218996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,7,0.02701759934425354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,15,0.03707680106163025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,15,0.02706719934940338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,31,0.037118399143218996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,63,0.037238401174545285
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,63,0.02688960134983063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,127,0.03912799954414368
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,127,0.026979199051856993
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,255,0.048214399814605714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,255,0.03165279924869537
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,511,0.08252639770507812
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,1,0.03916319906711578
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,3,0.04538559913635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,3,0.03919520080089569
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,7,0.045347198843955994
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,7,0.03918080031871796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,1,0.0271263986825943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,15,0.03904480040073395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,31,0.045337599515914914
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,31,0.03910239934921265
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,31,0.027809599041938783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,63,0.047502401471138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,63,0.03923520147800445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,127,0.05151200294494629
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,255,0.08403679728507996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,255,0.05130400061607361
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,1,0.01459839940071106
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,1,0.014574399590492249
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,3,0.01465120017528534
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,1,0.0454255998134613
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,3,0.014523200690746307
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,7,0.01648319959640503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,7,0.014577600359916686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,15,0.016417600214481354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,15,0.014577600359916686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,31,0.01648160070180893
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,31,0.01454399973154068
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,63,0.01659359931945801
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,63,0.014611199498176575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,127,0.016710400581359863
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,127,0.014617599546909332
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,255,0.016190400719642638
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,255,0.014601600170135499
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,511,0.014596800506114959
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,511,0.012569600343704223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,1023,0.012435200065374375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,2047,0.012518399953842163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,2047,0.01353919953107834
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,4095,0.014548799395561219
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,4095,0.012495999783277511
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,1,0.01242400035262108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,1,0.012467200309038163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,127,0.04206719994544983
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,3,0.012470400333404541
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,3,0.012417600303888322
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,7,0.01255359947681427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,7,0.01252480000257492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,15,0.01252480000257492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,15,0.012494400143623352
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,31,0.01257600039243698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,31,0.0124208003282547
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,63,0.012574400007724761
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,63,0.012587200105190276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,127,0.01255200058221817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,127,0.012559999525547028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,1023,0.012591999769210816
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,15,0.04531840085983276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,255,0.012532800436019897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,511,0.012163200229406358
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,511,0.011681599915027619
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,1023,0.010657600313425063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,1023,0.012177599966526032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,2047,0.012691199779510498
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,2047,0.01263359934091568
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,4095,0.014724799990653991
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,4095,0.014528000354766845
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,1,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,1,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,3,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,3,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,7,0.010567999631166457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,15,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,15,0.010592000186443329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,31,0.010555200278759003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,31,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,63,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,63,0.010679999738931656
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,127,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,255,0.012587200105190276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,127,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,255,0.010622400045394897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,511,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,511,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,1023,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,1023,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,2047,0.011798399686813354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,2047,0.010660800337791442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,4095,0.014591999351978302
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,1,0.010422399640083313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,7,0.01040000021457672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,1,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,3,0.010313600301742554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,3,0.010440000146627427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,7,0.010377600044012069
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,7,0.008532799780368805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,15,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,15,0.008560000360012055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,31,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,31,0.008526399731636047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,63,0.010289599746465683
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,63,0.009486400336027146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,127,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,127,0.009654399752616883
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,255,0.01032480001449585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,255,0.00941760018467903
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,511,0.010401599854230881
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,511,0.008851200342178345
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,1023,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,1023,0.0103472001850605
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,2047,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,2047,0.011796800047159195
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,4095,0.027217599749565124
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,4095,0.014582400023937226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,255,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,1,0.026374399662017822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,1,0.024676799774169922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,3,0.02654399871826172
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,7,0.0268640011548996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,7,0.022972799837589264
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,3,0.023003199696540834
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,15,0.026316800713539125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,15,0.024700799584388734
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,31,0.026342400908470155
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,63,0.026844799518585205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,63,0.023043200373649597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,127,0.02783840000629425
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,127,0.02295999974012375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,1,0.05318400263786316
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,1,0.04731360077857971
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,3,0.05305439829826355
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,3,0.047305598855018616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,7,0.053329598903656
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,7,0.04729759991168976
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,15,0.05304480195045471
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,15,0.04730879962444305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,31,0.05338720083236694
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,31,0.0473471999168396
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,63,0.05498719811439514
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,63,0.04737919867038727
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,127,0.057968002557754514
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,127,0.04891040027141571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,1,0.06586080193519592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,1,0.04538240134716034
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,3,0.06580640077590942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,3,0.04538240134716034
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,7,0.06582559943199158
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,7,0.046500799059867856
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,15,0.06587200164794922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,15,0.04620639979839325
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,31,0.06742560267448425
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,31,0.02314399927854538
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,31,0.04737280011177063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,63,0.06660320162773133
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,63,0.04740320146083832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,127,0.068094402551651
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,127,0.048860800266265866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,4095,0.01252640038728714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,1,0.08355839848518372
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,1,0.06988800168037415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,3,0.08235840201377868
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,3,0.06996960043907166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,7,0.08422880172729492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,7,0.07010560035705567
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,15,0.08468480110168457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,15,0.06992160081863404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,31,0.08454560041427613
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,63,0.08602240085601806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,63,0.07137119770050049
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,127,0.09279199838638305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,1,0.043398401141166686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,127,0.07408000230789184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,1,0.03922080099582672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,3,0.04350079894065857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,3,0.03926720023155213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,7,0.043489599227905275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,7,0.039371201395988466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,15,0.039427199959754945
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,31,0.04576959908008575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,31,0.03937439918518067
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,63,0.04681600034236908
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,63,0.03922719955444336
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,1,0.09688000082969665
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,1,0.08645280003547669
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,3,0.09776319861412049
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,3,0.08634399771690368
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,7,0.09873279929161072
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,7,0.08652480244636536
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,15,0.08650239706039428
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,31,0.1021664023399353
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,31,0.08752319812774659
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,63,0.1034208059310913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,63,0.08917279839515686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,1,0.12333600521087647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,1,0.08425599932670594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,3,0.12322560548782349
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,3,0.08427519798278808
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,7,0.12376480102539063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,7,0.08422560095787049
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,15,0.12412480115890503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,15,0.08426240086555481
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,15,0.09921600222587586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,31,0.08492640256881714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,15,0.04352479875087738
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,63,0.12392799854278565
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,63,0.08484960198402405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,1,0.15839840173721315
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,1,0.13122719526290894
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,3,0.13144479990005492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,7,0.15810079574584962
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,7,0.13137919902801515
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,15,0.1582479953765869
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,31,0.07060480117797852
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,15,0.13194719552993775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,31,0.15878720283508302
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,31,0.13279199600219727
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,63,0.1604624032974243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,31,0.12382080554962158
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,1,0.016545599699020384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,1,0.014591999351978302
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,3,0.01656640022993088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,3,0.014683200418949128
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,7,0.014612799882888794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,7,0.014587199687957764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,15,0.014683200418949128
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,15,0.014734399318695069
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,31,0.014767999947071075
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,31,0.01560640037059784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,63,0.015276800096035003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,63,0.01470080018043518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,127,0.016550399363040924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,127,0.014603200554847717
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,255,0.015291200578212738
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,255,0.01467680037021637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,511,0.014392000436782838
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,511,0.01356319934129715
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,1023,0.012566399574279786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,1023,0.01249919980764389
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,2047,0.012723200023174286
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,2047,0.012585599720478059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,4095,0.01494079977273941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,4095,0.014694400131702423
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,1,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,1,0.01223680004477501
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,3,0.010595200210809707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,3,0.012014400213956833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,7,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,7,0.010824000090360641
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,15,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,15,0.010673599690198899
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,31,0.012438400089740754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,31,0.012465599924325943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,63,0.13351839780807495
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,63,0.012508800625801087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,63,0.01141280010342598
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,127,0.012459199875593185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,127,0.012251199781894683
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,255,0.012491200119256973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,255,0.010592000186443329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,511,0.010585600137710571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,511,0.010555200278759003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,1023,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,2047,0.01266079992055893
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,3,0.15822559595108032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,2047,0.011923199892044068
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,4095,0.014560000598430633
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,4095,0.013628800213336945
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,1,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,3,0.010606399923563003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,3,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,7,0.01061599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,7,0.010592000186443329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,15,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,15,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,31,0.010654400289058685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,31,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,63,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,63,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,127,0.010660800337791442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,127,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,255,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,1,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,255,0.010892800241708755
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,511,0.010571199655532836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,1023,0.012388800084590913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,1023,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,2047,0.01398559957742691
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,2047,0.01255040019750595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,4095,0.029264000058174134
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,4095,0.014731200039386749
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,1,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,1,0.008561599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,3,0.008462399989366532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,3,0.00846880003809929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,7,0.008488000184297562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,15,0.008504000306129456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,15,0.008505599945783615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,31,0.00860000029206276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,31,0.00857279971241951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,1023,0.012276799976825714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,63,0.008470399677753449
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,127,0.008561599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,127,0.008584000170230865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,255,0.00849440023303032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,255,0.008606400340795517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,511,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,511,0.010471999645233154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,1023,0.012539200484752655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,1023,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,2047,0.026900801062583923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,2047,0.013121600449085235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,7,0.008463999629020691
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,4095,0.027031999826431275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,1,0.012604799866676331
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,1,0.012468799948692322
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,3,0.012379200011491776
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,3,0.012432000041007996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,7,0.012451200187206269
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,63,0.00851999968290329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,7,0.012459199875593185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,15,0.01271200031042099
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,15,0.01249919980764389
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,31,0.012479999661445617
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,31,0.012479999661445617
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,63,0.012569600343704223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,63,0.012388800084590913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,127,0.012544000148773193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,255,0.012641599774360657
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,255,0.012603199481964112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,511,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,511,0.008876799792051315
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,511,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,1023,0.012627199292182922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,2047,0.014606399834156037
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,2047,0.014644800126552582
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,4095,0.0297791987657547
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,4095,0.04503200054168701
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,4095,0.01867839992046356
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,8191,0.04732959866523743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,8191,0.031089600920677186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,1,0.010080000013113022
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,1,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,3,0.00852160006761551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,3,0.008531200140714646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,7,0.008476799726486206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,7,0.008563199639320373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,127,0.012548799812793731
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,15,0.009833600372076035
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,31,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,31,0.008532799780368805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,63,0.009806399792432785
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,63,0.008604799956083297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,127,0.010308799892663955
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,127,0.010351999849081039
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,255,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,255,0.009892799705266953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,511,0.010819199681282043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,511,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,15,0.008504000306129456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,1023,0.012478400021791458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,1023,0.01064319983124733
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,2047,0.027697598934173583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,2047,0.014640000462532044
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,4095,0.04543200135231018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,4095,0.028095999360084535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,8191,0.07937440276145935
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,8191,0.04564639925956726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,1,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,3,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,3,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,7,0.010708799958229065
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,7,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,15,0.010636799782514573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,1023,0.01255040019750595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,15,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,31,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,31,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,63,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,63,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,127,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,127,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,255,0.010545600205659866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,511,0.012590399384498597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,1023,0.027639999985694885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,511,0.011095999926328658
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,1023,0.014590400457382201
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,2047,0.046291199326515195
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,2047,0.028907200694084166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,4095,0.04631839990615845
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,8191,0.15026400089263917
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,8191,0.08148319721221924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,1,0.008995199948549271
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,1,0.012433599680662155
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,3,0.012489599734544754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,3,0.012515200674533844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,7,0.012619200348854064
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,7,0.011670400202274323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,15,0.01252799928188324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,15,0.011476799845695496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,31,0.01252799928188324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,255,0.01043199971318245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,31,0.01247360035777092
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,63,0.012656000256538392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,63,0.012695999443531036
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,127,0.012614400684833526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,127,0.012535999715328216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,255,0.012505599856376648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,511,0.02843039929866791
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,511,0.01459999978542328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,1023,0.0459199994802475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,1023,0.02921760082244873
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,2047,0.08152959942817688
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,1,0.012761600315570831
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,2047,0.046937599778175354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,4095,0.15236480236053468
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,8191,0.29490079879760744
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,8191,0.16027679443359374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,1,0.010606399923563003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,1,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,255,0.012574400007724761
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,3,0.011416000127792359
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,3,0.012443199753761292
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,7,0.012398400157690049
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,7,0.01241919994354248
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,15,0.0118367999792099
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,15,0.012516799569129943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,31,0.012622399628162384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,4095,0.08060160279273987
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,31,0.012430399656295776
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,63,0.011662399768829346
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,127,0.012503999471664428
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,63,0.012486399710178375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,127,0.010582400113344192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,255,0.012572799623012543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,255,0.012556800246238708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,511,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,1023,0.011664000153541566
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,1023,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,2047,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,4095,0.012675200402736665
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,4095,0.012625600397586822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,4095,0.08481760025024414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,8191,0.014668799936771393
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,8191,0.014590400457382201
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,1,0.010552000254392624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,1,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,3,0.010555200278759003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,3,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,7,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,7,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,15,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,15,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,31,0.010633599758148194
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,31,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,63,0.0106175996363163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,63,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,127,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,511,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,127,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,255,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,511,0.010665600001811982
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,255,0.01058719977736473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,511,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,1023,0.010609599947929382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,1023,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,2047,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,2047,0.01231359988451004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,4095,0.012695999443531036
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,8191,0.014777599275112152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,8191,0.014732800424098969
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,1,0.010911999642848969
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,1,0.008631999790668487
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,3,0.008550400286912918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,3,0.008591999858617782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,7,0.008561599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,7,0.008641599863767623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,15,0.00878399983048439
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,15,0.00856959968805313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,2047,0.012611199915409089
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,31,0.010401599854230881
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,63,0.010585600137710571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,63,0.008607999980449676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,127,0.010468800365924836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,4095,0.012588800489902496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,127,0.009267199784517288
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,255,0.008902399986982345
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,255,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,511,0.009156800061464309
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,511,0.010681600123643876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,1023,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,1023,0.009803199768066406
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,2047,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,4095,0.01250240057706833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,4095,0.010630399733781815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,8191,0.014535999298095703
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,8191,0.014481599628925323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,1,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,1,0.00857279971241951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,3,0.010395199805498124
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,3,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,7,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,7,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,15,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,31,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,31,0.010393600165843963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,63,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,31,0.008606400340795517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,63,0.010406400263309478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,127,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,255,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,255,0.009436800330877303
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,511,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,511,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,1023,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,1023,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,2047,0.010582400113344192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,2047,0.010625600069761276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,2047,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,4095,0.012507200241088867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,4095,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,8191,0.014560000598430633
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,8191,0.012556800246238708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,1,0.012535999715328216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,1,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,3,0.012529599666595458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,3,0.011392000317573547
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,7,0.012521600723266602
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,7,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,15,0.012564800679683685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,15,0.011681599915027619
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,31,0.012462399899959564
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,31,0.010758399963378906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,63,0.012537600100040435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,63,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,127,0.012520000338554382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,127,0.010564800351858139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,255,0.012624000012874604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,255,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,511,0.012193600088357926
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,511,0.010664000362157821
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,1023,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,1023,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,2047,0.012545600533485413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,2047,0.010681600123643876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,4095,0.012571200728416443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,4095,0.012579199671745301
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,8191,0.014579200744628906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,8191,0.014833599328994751
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,1,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,1,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,3,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,3,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,7,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,7,0.011019200086593628
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,15,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,15,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,31,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,31,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,63,0.010564800351858139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,127,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,63,0.010678400099277497
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,127,0.010664000362157821
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,127,0.010564800351858139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,255,0.01072319969534874
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,255,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,511,0.010739199817180634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,1023,0.010633599758148194
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,1023,0.010734400153160096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,2047,0.012608000636100769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,2047,0.012223999947309494
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,4095,0.012758399546146392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,4095,0.012718400359153748
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,15,0.0103472001850605
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,8191,0.01494240015745163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,1,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,8191,0.014724799990653991
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,1,0.008575999736785888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,3,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,3,0.008640000224113464
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,7,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,7,0.008687999844551087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,15,0.010552000254392624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,15,0.008620800077915191
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,31,0.010639999806880952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,31,0.00867680013179779
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,63,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,63,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,127,0.009694399684667588
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,127,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,255,0.010356800258159637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,511,0.010567999631166457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,255,0.009369599819183349
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,511,0.010542400181293488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,511,0.01053439974784851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,1023,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,2047,0.010542400181293488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,1023,0.010876800119876861
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,4095,0.012518399953842163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,4095,0.010728000104427338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,8191,0.01483200043439865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,8191,0.012729600071907043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,1,0.010564800351858139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,3,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,1,0.009804800152778625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,7,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,7,0.01074720025062561
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,15,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,15,0.01067200005054474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,31,0.010366400331258773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,63,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,31,0.010705599933862687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,63,0.0117807999253273
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,127,0.01072319969534874
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,127,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,255,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,255,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,511,0.010630399733781815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,511,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,1023,0.010755199939012527
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,1023,0.010406400263309478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,2047,0.010971199721097946
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,4095,0.012727999687194824
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,4095,0.011289600282907486
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,8191,0.014846399426460266
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,8191,0.012750400602817536
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,2047,0.010673599690198899
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,1,0.011787199974060058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,1,0.012039999663829803
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,3,0.012006399780511856
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,3,0.012388800084590913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,3,0.008510400354862214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,7,0.012432000041007996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,15,0.010552000254392624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,15,0.012483199685811996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,31,0.010681600123643876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,31,0.010702399909496308
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,63,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,63,0.010705599933862687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,127,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,127,0.010831999778747558
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,255,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,255,0.010619200021028518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,511,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,511,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,1023,0.010608000308275222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,1023,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,2047,0.012639999389648438
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,2047,0.012521600723266602
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,2047,0.010654400289058685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,4095,0.014596800506114959
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,4095,0.014528000354766845
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,8191,0.029065600037574767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,1,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,8191,0.018747200071811677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,1,0.010710400342941285
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,3,0.010676799714565277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,3,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,7,0.010683199763298035
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,7,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,15,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,7,0.01207360029220581
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,31,0.010761599987745285
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,63,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,63,0.010556799918413162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,127,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,127,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,255,0.010569600015878677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,255,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,511,0.010648000240325927
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,511,0.010622400045394897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,1023,0.01093439981341362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,1023,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,2047,0.012646399438381195
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,15,0.010622400045394897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,2047,0.012577599287033081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,4095,0.028999999165534973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,4095,0.01465280055999756
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,8191,0.04543519914150238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,8191,0.02911840081214905
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,1,0.00878399983048439
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,1,0.008508799970149994
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,3,0.008449599891901017
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,7,0.00849440023303032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,7,0.00843520015478134
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,15,0.0091839998960495
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,15,0.00852160006761551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,31,0.009038399904966354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,31,0.008513599634170532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,63,0.01037440001964569
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,127,0.010089600086212158
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,127,0.008473599702119828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,255,0.009814400225877762
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,255,0.008692800253629684
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,511,0.010603199899196624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,511,0.009398400038480758
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,1023,0.01255200058221817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,1023,0.010657600313425063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,2047,0.027673599123954774
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,31,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,2047,0.013143999874591828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,3,0.008975999802350998
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,4095,0.02826719880104065
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,8191,0.0795408010482788
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,8191,0.0455808013677597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,1,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,1,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,3,0.010592000186443329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,3,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,7,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,63,0.008587200194597244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,7,0.010396800190210342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,15,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,31,0.010385599732398988
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,31,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,63,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,63,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,127,0.01067039966583252
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,127,0.010281600058078766
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,255,0.010716799646615982
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,255,0.0091839998960495
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,511,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,511,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,1023,0.027580800652503967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,1023,0.014480000734329224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,2047,0.04540480077266693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,2047,0.028935998678207397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,4095,0.04534560143947601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,4095,0.08031839728355408
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,4095,0.047470399737358095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,1,0.012558400630950928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,8191,0.14973599910736085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,8191,0.08440160155296325
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,1,0.01268479973077774
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,3,0.011419200152158738
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,7,0.01252480000257492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,7,0.011059200018644333
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,15,0.01255200058221817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,15,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,31,0.012545600533485413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,31,0.01244800016283989
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,63,0.012591999769210816
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,63,0.012547199428081513
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,127,0.01252480000257492
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,127,0.01247519999742508
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,255,0.012596799433231354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,255,0.012478400021791458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,511,0.011657600104808808
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,511,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,1023,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,1023,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,2047,0.012539200484752655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,2047,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,4095,0.012644800543785095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,4095,0.012569600343704223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,8191,0.015915200114250183
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,8191,0.014686399698257446
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,1,0.010526400059461594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,3,0.012518399953842163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,15,0.012444800138473511
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,3,0.010659199953079224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,3,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,7,0.01067200005054474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,7,0.010390400141477584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,15,0.010363200306892395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,31,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,31,0.010398399829864503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,63,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,63,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,127,0.010471999645233154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,127,0.01035040020942688
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,255,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,255,0.010440000146627427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,511,0.010377600044012069
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,511,0.010471999645233154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,1023,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,1023,0.010470400005578995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,2047,0.012481600046157837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,2047,0.012116800248622894
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,4095,0.012571200728416443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,4095,0.01255040019750595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,8191,0.014529600739479065
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,8191,0.014611199498176575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,1,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,15,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,1,0.009617599844932555
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,3,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,3,0.009598399698734283
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,7,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,7,0.008611200004816055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,15,0.011007999628782272
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,31,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,63,0.011100800335407257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,31,0.010735999792814255
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,63,0.008529599756002426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,127,0.010841599851846694
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,127,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,255,0.010419200360774993
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,511,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,511,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,1023,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,1023,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,2047,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,2047,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,4095,0.012577599287033081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,1,0.010596799850463866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,4095,0.012511999905109405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,8191,0.014678399264812469
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,8191,0.012636800110340119
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,1,0.008478400111198426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,1,0.00844319984316826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,3,0.008511999994516373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,3,0.008483199775218964
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,7,0.008505599945783615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,15,0.008563199639320373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,15,0.009495999664068222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,15,0.008603200316429138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,31,0.009652800112962722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,31,0.008526399731636047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,63,0.009569600224494934
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,63,0.008551999926567078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,127,0.010340800136327743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,127,0.00854559987783432
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,255,0.009591999650001525
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,255,0.010943999886512757
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,511,0.008587200194597244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,511,0.008851200342178345
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,1023,0.011159999668598175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,2047,0.010916800051927567
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,2047,0.010756800323724747
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,4095,0.012870399653911591
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,4095,0.010875199735164643
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,8191,0.027617600560188294
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,8191,0.01485760062932968
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,7,0.008462399989366532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,1,0.0108255997300148
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,1,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,3,0.010707200318574906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,3,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,7,0.010679999738931656
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,7,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,15,0.010744000226259232
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,15,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,31,0.010628800094127654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,63,0.010755199939012527
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,255,0.008524800091981888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,63,0.010660800337791442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,1023,0.010398399829864503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,127,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,127,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,255,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,255,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,511,0.012611199915409089
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,1023,0.014791999757289887
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,1023,0.014180800318717957
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,2047,0.02935200035572052
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,2047,0.017004799842834473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,4095,0.047624000906944276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,4095,0.030368000268936157
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,1,0.012603199481964112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,1,0.010660800337791442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,3,0.012001600116491318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,3,0.01071999967098236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,7,0.012195199728012085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,7,0.010702399909496308
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,511,0.01213119998574257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,15,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,31,0.010707200318574906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,31,0.010649599879980088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,63,0.010771200060844421
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,63,0.010662399977445603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,127,0.012591999769210816
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,127,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,255,0.012267199903726577
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,511,0.014718399941921234
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,31,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,511,0.012628799676895142
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,1023,0.028652799129486085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,2047,0.04526239931583405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,2047,0.02905279994010925
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,15,0.011902400106191636
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,4095,0.07999839782714843
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,1,0.012555199861526489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,1,0.012516799569129943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,3,0.012595200538635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,3,0.01255040019750595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,7,0.012532800436019897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,7,0.012545600533485413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,15,0.012537600100040435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,15,0.012665599584579468
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,31,0.012622399628162384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,31,0.012539200484752655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,63,0.012694400548934937
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,255,0.010779199749231338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,63,0.012479999661445617
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,127,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,127,0.012606400251388549
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,1023,0.015982399880886077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,255,0.012534399330615998
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,511,0.02902719974517822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,1023,0.04625599980354309
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,4095,0.047547200322151185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,2047,0.08223680257797242
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,2047,0.04841760098934174
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,4095,0.15402239561080933
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,4095,0.08336640000343323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,1,0.01704320013523102
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,1,0.016575999557971954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,3,0.016974399983882903
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,3,0.01658399999141693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,7,0.01688160002231598
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,7,0.01478080004453659
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,255,0.01297920048236847
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,15,0.016828800737857818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,511,0.014699199795722961
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,31,0.016838400065898894
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,15,0.014828799664974213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,31,0.014550399780273438
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,63,0.01679680049419403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,63,0.016705599427223206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,127,0.016735999286174773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,127,0.01656000018119812
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,255,0.02837119996547699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,1023,0.029259198904037477
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,255,0.014947199821472168
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,511,0.04542079865932465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,511,0.029043200612068176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,1023,0.08048959970474243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,1023,0.0477616012096405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,2047,0.08338559865951538
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,2047,0.149726402759552
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,1,0.014500799775123595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,4095,0.2867487907409668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,3,0.014571200311183929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,7,0.014575999975204468
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,7,0.01268800050020218
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,15,0.013785600662231445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,15,0.012638400495052337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,31,0.013980799913406372
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,31,0.012747199833393097
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,63,0.013711999356746673
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,63,0.01279039978981018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,127,0.014215999841690063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,127,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,255,0.014688000082969666
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,255,0.012775999307632447
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,511,0.01675360053777695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,4095,0.15824639797210693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,3,0.01276479959487915
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,1023,0.029707199335098265
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,1023,0.01878879964351654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,1,0.014707200229167938
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,1,0.012588800489902496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,3,0.014766399562358857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,3,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,7,0.0147599995136261
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,7,0.012729600071907043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,15,0.014740799367427827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,15,0.012726399302482604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,31,0.014694400131702423
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,31,0.012651200592517852
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,63,0.014688000082969666
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,1,0.012692800164222718
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,63,0.012780800461769104
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,127,0.014665600657463074
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,127,0.012740799784660339
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,255,0.014870400726795196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,255,0.012756800651550293
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,511,0.02895680069923401
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,511,0.01684480011463165
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,1023,0.04748800098896026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,1023,0.031020799279212953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,1,0.016916799545288085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,1,0.01544319987297058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,3,0.01679680049419403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,3,0.014703999459743499
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,7,0.01687999963760376
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,7,0.014721600711345673
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,15,0.01870400011539459
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,15,0.015491199493408204
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,63,0.01804639995098114
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,31,0.016648000478744505
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,63,0.0166143998503685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,127,0.018028800189495087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,127,0.016571199893951415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,255,0.028604799509048463
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,511,0.045952001214027406
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,511,0.029416000843048094
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,1023,0.08210240006446838
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,511,0.014609600603580474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,1,0.026881599426269533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,1,0.022782400250434875
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,3,0.02693600058555603
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,3,0.022888000309467315
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,7,0.026919999718666078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,7,0.022916799783706664
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,15,0.02694239914417267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,31,0.016859200596809388
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,15,0.023076799511909486
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,31,0.026998400688171387
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,31,0.022867199778556824
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,63,0.02704479992389679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,63,0.02285120040178299
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,127,0.030950400233268737
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,127,0.02289759963750839
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,255,0.046188798546791074
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,255,0.016700799763202667
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,255,0.030276799201965333
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,511,0.08126239776611328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,511,0.047624000906944276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,1023,0.14994879961013793
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,1,0.0191551998257637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,1023,0.08338080048561096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,1,0.018585599958896637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,7,0.018785600364208222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,7,0.018518400192260743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,1023,0.04782879948616028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,15,0.018699200451374055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,15,0.018747200071811677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,31,0.01879359930753708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,31,0.018768000602722167
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,63,0.018721599876880646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,63,0.018775999546051025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,127,0.018838399648666383
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,255,0.018908800184726716
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,255,0.01866399943828583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,511,0.03208959996700287
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,511,0.022782400250434875
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,1,0.022115199267864226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,1,0.0185248002409935
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,3,0.022252799570560457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,3,0.018641600012779237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,7,0.022572800517082214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,7,0.01867839992046356
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,15,0.022438399493694305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,15,0.01860959976911545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,3,0.01889760047197342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,31,0.022767999768257143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,31,0.01873439997434616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,63,0.022758400440216063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,63,0.0186271995306015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,127,0.02279040068387985
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,127,0.018612800538539885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,255,0.02892639935016632
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,255,0.018636800348758698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,511,0.04736000001430511
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,511,0.029721599817276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,1,0.027054399251937866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,127,0.018772800266742707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,3,0.02705279886722565
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,3,0.022968000173568724
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,7,0.02697120010852814
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,7,0.023107199370861052
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,15,0.027116799354553224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,15,0.024897600710391998
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,31,0.027089598774909972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,31,0.025011199712753295
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,3,0.016841599345207216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,63,0.027131199836730957
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,63,0.022912000119686127
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,127,0.022910399734973906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,255,0.046321600675582886
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,255,0.029606398940086365
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,511,0.08237119913101196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,1,0.022913600504398345
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,511,0.0475055992603302
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,1,0.04536960124969482
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,1,0.03935680091381073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,3,0.04535839855670929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,3,0.03911679983139038
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,7,0.039262399077415466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,15,0.045449599623680115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,15,0.03734399974346161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,31,0.04547359943389893
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,31,0.03746559917926788
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,63,0.04761120080947876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,63,0.03923520147800445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,127,0.031065601110458373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,127,0.051412802934646604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,127,0.04164640009403229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,255,0.0825760006904602
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,255,0.0513759970664978
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,511,0.0842191994190216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,1,0.012727999687194824
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,1,0.012566399574279786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,3,0.012532800436019897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,3,0.012614400684833526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,7,0.012547199428081513
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,15,0.012559999525547028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,7,0.04524959921836853
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,15,0.010728000104427338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,31,0.012559999525547028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,31,0.011107199639081956
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,63,0.012534399330615998
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,63,0.010744000226259232
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,127,0.012544000148773193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,127,0.012220799922943115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,255,0.012539200484752655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,255,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,511,0.01111999973654747
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,511,0.010625600069761276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,1023,0.01061599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,1023,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,2047,0.012532800436019897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,2047,0.012566399574279786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,4095,0.014604799449443817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,4095,0.012715199589729309
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,8191,0.014632000029087067
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,1,0.010585600137710571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,1,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,3,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,3,0.010603199899196624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,511,0.15195200443267823
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,7,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,7,0.01263359934091568
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,15,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,31,0.010699199885129929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,31,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,63,0.010633599758148194
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,63,0.010673599690198899
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,127,0.010644800215959548
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,127,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,255,0.010608000308275222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,255,0.010556799918413162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,8191,0.016070400178432465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,511,0.010756800323724747
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,511,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,1023,0.012030400335788727
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,1023,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,2047,0.012689599394798278
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,2047,0.010999999940395355
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,4095,0.014608000218868256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,4095,0.012574400007724761
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,8191,0.016641600430011748
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,8191,0.014587199687957764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,1,0.009825599938631057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,1,0.00843999981880188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,3,0.009780800342559815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,3,0.008566399663686752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,15,0.010608000308275222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,15,0.008544000238180161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,15,0.008577600121498108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,31,0.010273600369691849
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,31,0.008689600229263305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,63,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,63,0.008535999804735184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,127,0.008564800024032593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,7,0.01053439974784851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,127,0.008574400097131729
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,255,0.008564800024032593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,7,0.008555199950933456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,511,0.00851840004324913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,511,0.00851840004324913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,1023,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,1023,0.009297599643468856
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,2047,0.01061440035700798
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,2047,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,4095,0.01268800050020218
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,4095,0.010592000186443329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,8191,0.027160000801086426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,8191,0.014420799911022186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,1,0.009073600172996521
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,1,0.008560000360012055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,3,0.008715199679136277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,3,0.00873119980096817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,7,0.01048479974269867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,7,0.009646400064229965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,15,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,15,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,7,0.009259200096130371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,31,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,31,0.008675199747085572
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,63,0.010779199749231338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,127,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,255,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,127,0.010916800051927567
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,255,0.00968799963593483
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,511,0.011132799834012986
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,511,0.008614400029182434
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,1023,0.01098719984292984
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,1023,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,2047,0.01064160019159317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,4095,0.02744159996509552
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,4095,0.014571200311183929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,8191,0.04543839991092682
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,8191,0.02868640124797821
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,1,0.030972799658775328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,1,0.0853056013584137
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,3,0.031068798899650574
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,3,0.029025599360466003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,255,0.008528000116348267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,7,0.03097760081291199
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,15,0.03089280128479004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,63,0.01010880023241043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,15,0.0288783997297287
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,31,0.030928000807762146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,31,0.02889440059661865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,63,0.030955201387405394
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,63,0.028910401463508605
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,127,0.030956798791885377
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,127,0.028880000114440918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,255,0.03358240127563476
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,255,0.029003199934959412
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,2047,0.012937599420547485
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,1,0.028830400109291075
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,3,0.03703519999980927
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,3,0.02887200117111206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,7,0.03700959980487824
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,7,0.02805599868297577
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,15,0.03702079951763153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,15,0.028169599175453187
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,31,0.03709439933300018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,31,0.028624001145362853
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,63,0.03723039925098419
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,63,0.02869440019130707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,127,0.03835200071334839
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,255,0.04827040135860443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,255,0.03276160061359405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,7,0.028630399703979494
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,1,0.03909600079059601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,3,0.04532960057258606
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,3,0.03918719887733459
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,7,0.045428800582885745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,7,0.039262399077415466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,15,0.045263999700546266
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,15,0.03922559916973114
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,1,0.03532319962978363
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,31,0.03919839859008789
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,63,0.04743199944496155
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,63,0.03914560079574585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,127,0.051420801877975465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,127,0.04236319959163666
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,255,0.08399519920349122
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,127,0.028697600960731505
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,255,0.05000159740447998
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,1,0.08233280181884765
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,1,0.06799039840698243
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,3,0.08220319747924805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,3,0.0678816020488739
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,1,0.04539200067520142
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,7,0.08233280181884765
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,7,0.06780480146408081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,15,0.08296480178833007
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,15,0.07000640034675598
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,31,0.08259680271148681
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,31,0.06983680129051209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,63,0.08471360206604003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,63,0.06988000273704528
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,127,0.09173759818077087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,127,0.07216479778289794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,255,0.15526399612426758
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,255,0.0920960009098053
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,31,0.04645119905471802
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,1,0.05316799879074097
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,3,0.05151039958000183
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,1,0.047926399111747744
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,7,0.05152959823608398
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,3,0.04943679869174957
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,7,0.047470399737358095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,15,0.051507198810577394
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,15,0.04939680099487305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,31,0.05338559746742248
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,63,0.05499839782714844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,63,0.048640000820159915
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,127,0.05881919860839844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,127,0.04964320063591003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,1,0.06589279770851135
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,1,0.04758400022983551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,3,0.06604480147361755
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,3,0.047972801327705386
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,7,0.0660207986831665
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,7,0.04742720127105713
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,15,0.06602720022201539
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,15,0.047358399629592894
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,31,0.066048002243042
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,63,0.06638399958610534
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,63,0.04948320090770721
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,127,0.06822720170021057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,127,0.04998559951782226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,1,0.0824400007724762
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,1,0.07036479711532592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,3,0.08256800174713134
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,3,0.07032639980316162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,7,0.08244959712028503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,7,0.07127519845962524
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,15,0.08452799916267395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,15,0.07008320093154907
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,31,0.047435200214385985
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,31,0.07195039987564086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,31,0.08573120236396789
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,63,0.08453919887542724
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,63,0.07389600276947021
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,127,0.09503520131111146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,1,0.1560960054397583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,1,0.13184640407562256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,3,0.15628639459609986
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,3,0.13170080184936522
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,7,0.15696320533752442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,7,0.13157600164413452
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,31,0.04745120108127594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,31,0.15622559785842896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,15,0.13159040212631226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,31,0.13125920295715332
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,63,0.15905760526657103
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,63,0.13158559799194336
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,127,0.17293599843978882
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,1,0.012503999471664428
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,1,0.01241919994354248
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,3,0.012478400021791458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,3,0.012606400251388549
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,7,0.012492799758911132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,7,0.012478400021791458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,127,0.07432159781455994
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,15,0.012582400441169738
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,15,0.01255359947681427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,31,0.012627199292182922
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,31,0.012513600289821625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,63,0.012649600207805634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,63,0.01250240057706833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,127,0.012699200212955475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,127,0.012572799623012543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,255,0.01257600039243698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,255,0.01252799928188324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,511,0.01249919980764389
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,15,0.15725120306015014
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,511,0.012409599870443344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,1023,0.012464000284671784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,1023,0.011577600240707397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,2047,0.012638400495052337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,2047,0.012521600723266602
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,4095,0.014672000706195832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,4095,0.014580799639225006
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,8191,0.017161600291728973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,8191,0.016553600132465363
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,1,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,3,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,7,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,127,0.13560479879379272
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,15,0.010416000336408614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,7,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,15,0.010470400005578995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,31,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,31,0.010358399897813796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,63,0.01067200005054474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,127,0.010635200142860412
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,127,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,255,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,1,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,3,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,255,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,511,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,511,0.008575999736785888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,1023,0.010715200006961823
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,1023,0.010689599812030793
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,2047,0.01250240057706833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,2047,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,4095,0.013870400190353394
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,4095,0.012582400441169738
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,63,0.010382399708032609
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,8191,0.028825598955154418
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,8191,0.0146479994058609
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,1,0.010316800326108932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,1,0.008742400258779526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,3,0.008617600053548813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,7,0.008479999750852585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,7,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,15,0.00883679986000061
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,31,0.008959999680519104
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,15,0.010636799782514573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,31,0.008408000320196151
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,63,0.010664000362157821
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,63,0.008473599702119828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,127,0.008697599917650223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,127,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,255,0.008905600011348724
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,511,0.010331200063228607
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,511,0.009081599861383438
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,1023,0.010470400005578995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,1023,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,2047,0.012572799623012543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,2047,0.010654400289058685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,4095,0.027028799057006836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,4095,0.014452800154685974
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,8191,0.04523519873619079
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,8191,0.02852959930896759
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,1,0.008510400354862214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,1,0.00846880003809929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,3,0.008511999994516373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,3,0.009787199646234512
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,7,0.009292799979448318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,3,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,15,0.009204799681901932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,15,0.008526399731636047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,31,0.008441600203514098
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,63,0.008432000130414962
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,63,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,127,0.008935999870300294
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,255,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,127,0.008446399867534638
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,255,0.008689600229263305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,255,0.008408000320196151
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,511,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,511,0.008508799970149994
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,1023,0.012443199753761292
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,1023,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,2047,0.027107200026512145
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,2047,0.012569600343704223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,4095,0.04523679912090302
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,4095,0.0269679993391037
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,8191,0.07999039888381958
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,8191,0.04519839882850647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,1,0.01067200005054474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,1,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,3,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,3,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,7,0.010652799904346467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,7,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,15,0.01064160019159317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,15,0.010526400059461594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,31,0.010625600069761276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,31,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,63,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,63,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,127,0.010620799660682679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,127,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,255,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,31,0.008827199786901474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,255,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,511,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,1023,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,1023,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,2047,0.012511999905109405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,2047,0.012582400441169738
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,4095,0.014616000652313232
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,8191,0.045817598700523376
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,8191,0.02906239926815033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,16383,0.08022400140762329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,16383,0.04692319929599762
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,1,0.008555199950933456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,1,0.00862239971756935
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,3,0.008582399785518646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,3,0.008644799888134002
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,7,0.008568000048398972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,7,0.008843199908733368
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,7,0.00846560001373291
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,15,0.009267199784517288
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,15,0.008489599823951722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,31,0.008588799834251403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,31,0.008575999736785888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,63,0.008579199761152267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,63,0.008531200140714646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,511,0.010731200128793717
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,127,0.010371199995279311
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,255,0.009675200283527374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,255,0.008542399853467941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,4095,0.028703999519348145
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,511,0.010636799782514573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,511,0.008432000130414962
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,1023,0.012460800260305405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,1023,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,2047,0.02699039876461029
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,2047,0.012577599287033081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,4095,0.04499199986457825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,4095,0.027739199995994567
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,8191,0.07835040092468262
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,8191,0.04543519914150238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,16383,0.1469104051589966
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,16383,0.07983199954032898
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,1,0.010449600219726563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,3,0.010409600287675857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,3,0.01032319962978363
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,7,0.010585600137710571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,7,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,15,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,15,0.010407999902963639
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,31,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,31,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,63,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,127,0.008587200194597244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,63,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,127,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,127,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,255,0.010625600069761276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,255,0.010387200117111205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,511,0.010699199885129929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,1023,0.026929599046707154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,1023,0.014556799829006196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,2047,0.0453247994184494
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,2047,0.028564798831939697
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,4095,0.08008959889411926
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,1,0.009364800155162811
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,8191,0.1489840030670166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,8191,0.08194079995155334
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,16383,0.28778719902038574
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,16383,0.15274560451507568
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,1,0.012531200051307678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,1,0.012443199753761292
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,3,0.012532800436019897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,511,0.012702399492263794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,3,0.01247360035777092
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,7,0.012579199671745301
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,7,0.01252799928188324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,15,0.012566399574279786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,15,0.012468799948692322
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,31,0.012593600153923034
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,63,0.012628799676895142
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,63,0.012433599680662155
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,127,0.012579199671745301
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,127,0.012432000041007996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,4095,0.046060800552368164
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,255,0.012563200294971466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,255,0.012567999958992004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,511,0.028537601232528687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,511,0.014710399508476257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,1023,0.04624319970607758
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,2047,0.08149120211601257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,2047,0.04728319942951202
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,4095,0.15274399518966675
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,4095,0.08452640175819397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,8191,0.29480319023132323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,8191,0.15875359773635864
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,31,0.012265600264072418
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,1,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,1,0.008774399757385254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,16383,0.31176960468292236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,16383,0.5797327995300293
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,3,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,7,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,7,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,1023,0.029017600417137145
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,15,0.010420800000429154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,31,0.010470400005578995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,31,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,63,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,63,0.01043360009789467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,127,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,127,0.010470400005578995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,255,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,255,0.010424000024795533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,511,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,511,0.01040000021457672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,1023,0.010567999631166457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,1023,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,2047,0.01247360035777092
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,2047,0.010407999902963639
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,4095,0.012566399574279786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,4095,0.011671999841928482
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,8191,0.014582400023937226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,8191,0.014507199823856353
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,16383,0.01858240067958832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,16383,0.01857440024614334
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,1,0.010366400331258773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,1,0.008633600175380706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,3,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,3,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,7,0.01032480001449585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,7,0.008555199950933456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,15,0.010361599922180175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,3,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,15,0.00865119993686676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,31,0.00841120034456253
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,31,0.010096000134944915
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,63,0.008499199897050858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,63,0.010238400101661682
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,15,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,127,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,255,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,255,0.010401599854230881
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,511,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,511,0.01034879982471466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,1023,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,1023,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,2047,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,4095,0.012545600533485413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,4095,0.010552000254392624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,8191,0.014500799775123595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,8191,0.01271039992570877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,16383,0.018636800348758698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,16383,0.018566399812698364
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,1,0.01037919968366623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,1,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,3,0.010406400263309478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,3,0.008641599863767623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,7,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,7,0.008551999926567078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,15,0.010380800068378448
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,15,0.008526399731636047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,31,0.010345599800348281
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,127,0.00851999968290329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,31,0.008612799644470214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,63,0.008564800024032593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,63,0.009431999921798707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,127,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,127,0.008823999762535095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,2047,0.010636799782514573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,255,0.00944959968328476
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,511,0.0104032002389431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,511,0.009225600212812424
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,1023,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,2047,0.01064319983124733
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,2047,0.010444799810647965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,4095,0.012534399330615998
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,8191,0.014560000598430633
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,8191,0.013660800457000733
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,16383,0.016697600483894348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,16383,0.01481119990348816
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,1,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,1,0.008486399799585343
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,3,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,3,0.00857120007276535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,7,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,7,0.008968000113964082
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,15,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,15,0.010396800190210342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,31,0.009787199646234512
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,255,0.01040479987859726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,31,0.00923840031027794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,63,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,127,0.010372799634933472
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,63,0.008553600311279297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,127,0.01034879982471466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,255,0.010358399897813796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,511,0.010398399829864503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,4095,0.010435199737548828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,511,0.010353600233793258
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,1023,0.010407999902963639
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,1023,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,2047,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,2047,0.010606399923563003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,4095,0.012457600235939026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,8191,0.01451520025730133
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,8191,0.012479999661445617
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,16383,0.0161408007144928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,16383,0.014550399780273438
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,1,0.010393600165843963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,1,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,3,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,3,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,7,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,7,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,15,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,15,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,31,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,31,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,63,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,1023,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,255,0.009668800234794616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,127,0.010700800269842149
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,127,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,255,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,255,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,4095,0.012544000148773193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,511,0.010768000036478043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,1023,0.010702399909496308
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,511,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,1023,0.01061440035700798
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,2047,0.012438400089740754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,4095,0.012676799297332763
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,2047,0.010608000308275222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,4095,0.012548799812793731
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,8191,0.014611199498176575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,8191,0.014612799882888794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,16383,0.018648000061511995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,16383,0.01871200054883957
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,1,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,3,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,1,0.0086496002972126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,3,0.00897120013833046
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,7,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,15,0.010633599758148194
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,15,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,63,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,31,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,31,0.008585599809885025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,63,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,63,0.008388800173997879
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,127,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,127,0.008535999804735184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,255,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,255,0.008459199965000153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,511,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,511,0.008505599945783615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,1023,0.01040479987859726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,1023,0.008460800349712371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,2047,0.010806400328874588
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,2047,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,4095,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,8191,0.014575999975204468
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,8191,0.012955200672149659
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,16383,0.01669279932975769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,16383,0.014579200744628906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,1,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,1,0.008505599945783615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,7,0.010371199995279311
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,3,0.009256000071763993
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,7,0.00902559980750084
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,7,0.008544000238180161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,15,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,15,0.009225600212812424
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,31,0.010440000146627427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,63,0.010416000336408614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,31,0.008934400230646133
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,63,0.008481600135564805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,127,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,255,0.008491200208663941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,255,0.008540800213813782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,511,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,511,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,1023,0.010582400113344192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,1023,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,2047,0.010552000254392624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,2047,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,4095,0.01257600039243698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,3,0.010367999970912933
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,4095,0.01067200005054474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,8191,0.012507200241088867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,16383,0.016633599996566772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,16383,0.01456640064716339
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,1,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,1,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,3,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,127,0.008563199639320373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,3,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,7,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,7,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,15,0.01043360009789467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,4095,0.012588800489902496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,15,0.008452799916267396
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,31,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,31,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,63,0.009118399769067764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,127,0.010364799946546554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,255,0.010388799756765366
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,255,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,511,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,511,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,1023,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,1023,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,8191,0.014510400593280792
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,2047,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,2047,0.010649599879980088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,4095,0.012651200592517852
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,4095,0.012492799758911132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,8191,0.014737600088119506
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,8191,0.012580800056457519
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,16383,0.02948479950428009
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,1,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,16383,0.016622400283813475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,1,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,3,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,3,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,7,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,63,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,7,0.010440000146627427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,127,0.010036800056695938
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,15,0.010608000308275222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,15,0.010694400221109391
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,31,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,31,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,63,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,127,0.011032000184059143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,127,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,255,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,511,0.010824000090360641
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,511,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,1023,0.01058719977736473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,1023,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,2047,0.01063840016722679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,2047,0.01063840016722679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,4095,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,4095,0.012664000689983367
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,8191,0.027937600016593934
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,8191,0.014753599464893342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,63,0.010552000254392624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,16383,0.045491200685501096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,1,0.010545600205659866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,16383,0.029128000140190125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,255,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,1,0.008568000048398972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,3,0.010567999631166457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,3,0.008488000184297562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,7,0.010526400059461594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,7,0.010844799876213073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,15,0.010393600165843963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,15,0.008558399975299835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,31,0.010372799634933472
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,63,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,63,0.009678400307893752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,127,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,127,0.010440000146627427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,255,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,511,0.008560000360012055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,1023,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,1023,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,2047,0.012529599666595458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,2047,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,4095,0.027081599831581114
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,4095,0.012863999605178833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,8191,0.04533919990062714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,8191,0.028417599201202393
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,16383,0.07949439883232116
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,16383,0.04552479982376099
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,1,0.008604799956083297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,1,0.008511999994516373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,31,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,3,0.008564800024032593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,3,0.008641599863767623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,7,0.008481600135564805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,7,0.008539199829101562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,15,0.008441600203514098
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,255,0.009900800138711929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,31,0.008502399921417237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,511,0.010393600165843963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,63,0.00854559987783432
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,63,0.008587200194597244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,127,0.008563199639320373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,127,0.008603200316429138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,255,0.008451200276613235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,255,0.008603200316429138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,511,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,511,0.010435199737548828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,1023,0.01255040019750595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,1023,0.010707200318574906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,2047,0.02741760015487671
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,2047,0.01414719969034195
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,4095,0.045307201147079465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,4095,0.027724799513816834
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,8191,0.07986720204353333
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,8191,0.045470398664474485
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,16383,0.14838720560073854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,16383,0.07992159724235534
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,1,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,3,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,3,0.009961599856615067
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,7,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,7,0.010040000081062317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,15,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,15,0.00852160006761551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,15,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,31,0.008398400247097015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,31,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,31,0.010976000130176545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,63,0.010420800000429154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,127,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,127,0.010407999902963639
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,255,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,255,0.008905600011348724
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,511,0.012603199481964112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,511,0.010691200196743012
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,1023,0.027297601103782654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,1023,0.014886400103569031
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,2047,0.04487360119819641
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,2047,0.027643200755119324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,4095,0.08011040091514587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,1,0.010582400113344192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,4095,0.046958398818969724
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,8191,0.1498095989227295
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,8191,0.0853712022304535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,16383,0.28848159313201904
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,16383,0.16376160383224486
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,1,0.011150400340557098
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,63,0.01061599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,3,0.010734400153160096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,3,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,7,0.010700800269842149
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,7,0.010654400289058685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,15,0.010737600177526474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,15,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,31,0.010659199953079224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,63,0.010742399841547012
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,63,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,127,0.010673599690198899
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,127,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,255,0.010790400207042694
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,255,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,511,0.010691200196743012
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,511,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,1023,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,2047,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,1,0.010556799918413162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,4095,0.012736000120639801
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,4095,0.012620800733566284
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,8191,0.014659200608730317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,8191,0.014696000516414643
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,31,0.010739199817180634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,16383,0.016841599345207216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,16383,0.01669279932975769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,1,0.00851840004324913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,1,0.008924800157546996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,3,0.008542399853467941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,3,0.008742400258779526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,7,0.008694399893283845
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,7,0.008908800035715102
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,15,0.00867839977145195
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,15,0.008743999898433686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,31,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,31,0.008848000317811966
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,63,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,63,0.008892799913883209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,127,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,127,0.01043199971318245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,255,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,511,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,511,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,1023,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,1023,0.010582400113344192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,2047,0.01048479974269867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,2047,0.010609599947929382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,2047,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,4095,0.01268800050020218
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,4095,0.012441600114107132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,8191,0.012531200051307678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,16383,0.016654400527477263
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,16383,0.014601600170135499
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,1,0.010072000324726105
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,1,0.008475200086832047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,3,0.010107199847698211
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,3,0.00843840017914772
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,7,0.010097599774599075
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,7,0.008491200208663941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,15,0.01040479987859726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,15,0.009616000205278396
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,255,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,31,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,63,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,63,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,127,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,127,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,255,0.010422399640083313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,1023,0.010790400207042694
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,255,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,511,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,511,0.009388799965381622
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,1023,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,1023,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,8191,0.014510400593280792
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,2047,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,2047,0.011510399729013443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,4095,0.012830400466918945
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,4095,0.012439999729394913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,8191,0.012787200510501862
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,16383,0.029212799668312073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,16383,0.01637759953737259
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,1,0.008459199965000153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,1,0.008664000034332275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,3,0.00838399976491928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,3,0.008617600053548813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,31,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,7,0.00851999968290329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,15,0.009344000369310379
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,15,0.008750399947166443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,31,0.00854559987783432
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,31,0.008505599945783615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,63,0.0084927998483181
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,63,0.008607999980449676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,127,0.008535999804735184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,127,0.00859680026769638
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,8191,0.01472959965467453
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,255,0.008585599809885025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,255,0.008568000048398972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,511,0.008580800145864487
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,511,0.008817599713802337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,1023,0.01061440035700798
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,7,0.008393599838018417
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,1023,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,2047,0.01061440035700798
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,2047,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,4095,0.012744000554084778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,4095,0.010684800148010255
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,8191,0.027006399631500245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,8191,0.014595200121402741
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,16383,0.02882240116596222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,16383,0.04541600048542023
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,1,0.009799999743700027
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,3,0.008550400286912918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,3,0.00862559974193573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,7,0.008563199639320373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,7,0.008606400340795517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,15,0.008641599863767623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,15,0.008631999790668487
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,31,0.008582399785518646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,31,0.008627200126647949
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,63,0.008665599673986436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,127,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,127,0.009355200082063675
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,255,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,255,0.009462399780750275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,511,0.010760000348091126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,511,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,1023,0.012705600261688233
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,1023,0.012590399384498597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,2047,0.027606400847434997
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,2047,0.014708800613880158
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,4095,0.045311999320983884
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,4095,0.02890079915523529
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,8191,0.07985439896583557
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,8191,0.045974400639533994
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,1,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,3,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,3,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,7,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,7,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,63,0.009462399780750275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,15,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,31,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,31,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,63,0.010569600015878677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,63,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,127,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,127,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,255,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,255,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,511,0.012544000148773193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,511,0.010630399733781815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,1023,0.027687999606132507
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,1023,0.014641599357128143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,1,0.008417599648237229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,1,0.01037919968366623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,2047,0.04582880139350891
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,2047,0.02794559895992279
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,4095,0.08017119765281677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,8191,0.14856959581375123
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,8191,0.08194559812545776
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,15,0.010608000308275222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,1,0.013065600395202636
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,3,0.01266079992055893
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,3,0.012503999471664428
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,7,0.012611199915409089
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,7,0.01242400035262108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,15,0.012641599774360657
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,15,0.012561599910259246
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,31,0.012567999958992004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,31,0.012521600723266602
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,63,0.012615999579429627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,63,0.012470400333404541
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,127,0.012782399356365205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,127,0.012459199875593185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,255,0.012600000202655792
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,255,0.012571200728416443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,511,0.028417599201202393
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,511,0.014703999459743499
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,1023,0.04704639911651611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,1023,0.029316800832748412
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,2047,0.08211359977722169
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,4095,0.15235840082168578
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,4095,0.08245919942855835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,4095,0.04617440104484558
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,8191,0.296891188621521
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,8191,0.15257439613342286
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,1,0.016953599452972413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,1,0.016710400581359863
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,3,0.017478400468826295
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,3,0.01679999977350235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,7,0.017020800709724428
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,7,0.016672000288963318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,15,0.017665599286556245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,2047,0.0480783998966217
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,31,0.01687840074300766
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,31,0.014787200093269347
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,63,0.0182559996843338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,63,0.01666239947080612
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,127,0.017817600071430205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,1,0.01249760016798973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,127,0.016828800737857818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,255,0.026956799626350402
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,255,0.016939200460910797
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,511,0.04524320065975189
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,1023,0.08003039956092835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,1023,0.04675360023975372
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,2047,0.14872640371322632
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,2047,0.08228319883346558
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,4095,0.2858416080474854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,15,0.01675039976835251
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,4095,0.15703840255737306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,1,0.010983999818563461
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,8191,0.5622799873352051
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,8191,0.30965280532836914
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,1,0.010953599959611893
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,3,0.012691199779510498
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,7,0.011075200140476226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,7,0.011044800281524658
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,15,0.011168000102043153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,511,0.02937600016593933
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,15,0.010974399745464325
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,31,0.01098880022764206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,31,0.01064160019159317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,63,0.012577599287033081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,127,0.01257600039243698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,127,0.010835199803113937
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,255,0.012091200053691863
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,255,0.010836800187826156
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,511,0.014665600657463074
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,511,0.012960000336170197
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,1023,0.02808000147342682
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,1023,0.01672320067882538
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,2047,0.029281601309776306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,1,0.012809599936008453
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,1,0.012507200241088867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,3,0.012619200348854064
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,3,0.012572799623012543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,7,0.012638400495052337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,7,0.012432000041007996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,15,0.012676799297332763
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,3,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,15,0.01250080019235611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,31,0.012569600343704223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,31,0.012577599287033081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,63,0.01250080019235611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,63,0.01250080019235611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,127,0.01250240057706833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,127,0.01257600039243698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,255,0.012588800489902496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,255,0.012478400021791458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,511,0.027897599339485168
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,63,0.010952000319957734
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,511,0.01480959951877594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,1023,0.04579679965972901
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,1023,0.029126399755477907
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,2047,0.04721759855747223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,1,0.016704000532627106
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,1,0.014619199931621552
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,3,0.016752000153064727
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,3,0.014580799639225006
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,7,0.016707199811935424
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,7,0.014679999649524688
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,2047,0.045814400911331175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,15,0.014667199552059173
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,15,0.016774399578571318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,31,0.016731199622154237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,31,0.015124799311161041
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,63,0.01467359960079193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,127,0.014748799800872802
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,255,0.027775999903678895
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,255,0.016622400283813475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,511,0.04565440118312836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,511,0.02905600070953369
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,1023,0.08031839728355408
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,1023,0.04705120027065277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,2047,0.14840480089187622
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,2047,0.08220959901809692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,1,0.0270224004983902
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,2047,0.08184000253677368
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,1,0.022998400032520294
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,3,0.022974400222301482
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,7,0.02714880108833313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,7,0.023094399273395537
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,127,0.016756799817085267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,3,0.026900801062583923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,15,0.022913600504398345
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,31,0.026833599805831908
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,31,0.022971199452877046
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,15,0.026998400688171387
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,63,0.02697120010852814
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,63,0.023004800081253052
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,127,0.03115679919719696
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,127,0.023259200155735016
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,255,0.045771199464797976
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,255,0.029524800181388856
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,511,0.080622398853302
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,511,0.04750719964504242
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,1023,0.14979039430618285
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,1023,0.08271679878234864
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,2047,0.15488959550857545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,2047,0.2863856077194214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,1,0.01478399932384491
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,1,0.013956800103187561
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,3,0.014764800667762756
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,3,0.014420799911022186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,7,0.01488800048828125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,7,0.013812799751758576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,15,0.014745600521564484
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,15,0.015248000621795654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,31,0.014894400537014008
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,31,0.01467519998550415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,63,0.014817599952220917
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,127,0.014497600495815277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,63,0.016684800386428833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,255,0.014884799718856812
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,255,0.01265919953584671
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,511,0.016734400391578676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,1023,0.04745280146598816
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,1023,0.03101919889450073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,1,0.01672160029411316
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,1,0.016705599427223206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,3,0.016702400147914888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,3,0.01666560024023056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,7,0.01664479970932007
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,7,0.016616000235080718
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,15,0.016710400581359863
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,15,0.016814400255680085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,31,0.01671839952468872
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,31,0.016657599806785585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,63,0.016675199568271636
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,63,0.016103999316692354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,127,0.01676799952983856
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,511,0.0290367990732193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,127,0.016297599673271178
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,255,0.028841599822044373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,255,0.01520799994468689
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,511,0.04621120095252991
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,511,0.028937599062919615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,1023,0.08047040104866028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,1023,0.047367998957633974
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,1,0.02704319953918457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,1,0.02279520034790039
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,3,0.02683199942111969
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,3,0.02295680046081543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,7,0.026976001262664796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,7,0.02292799949645996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,15,0.026846399903297423
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,15,0.022889600694179536
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,31,0.02704319953918457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,31,0.02295680046081543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,63,0.026984000205993654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,63,0.02295839935541153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,127,0.022881600260734557
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,255,0.045484799146652224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,255,0.029151999950408937
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,63,0.014684799313545226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,511,0.08024960160255432
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,511,0.04732959866523743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,1023,0.08220639824867249
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,1023,0.15011520385742189
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,1,0.04525440037250519
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,1,0.039129599928855896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,3,0.04524640142917633
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,3,0.03915359973907471
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,7,0.04528320133686066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,7,0.03916000127792359
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,15,0.04525760114192963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,31,0.04532319903373718
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,31,0.03902879953384399
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,63,0.047516798973083495
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,63,0.038515201210975646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,127,0.04975680112838745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,127,0.04021599888801575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,127,0.030983999371528625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,255,0.08082079887390137
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,255,0.05027040243148804
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,511,0.1498415946960449
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,511,0.08325120210647582
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,127,0.014643199741840363
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,1023,0.28696959018707274
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,1,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,1023,0.1546015977859497
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,3,0.010620799660682679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,7,0.010664000362157821
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,15,0.03920319974422455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,7,0.010603199899196624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,15,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,15,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,31,0.010608000308275222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,31,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,63,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,63,0.010596799850463866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,127,0.01067039966583252
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,127,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,255,0.010620799660682679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,1,0.010444799810647965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,3,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,511,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,1023,0.010609599947929382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,1023,0.010768000036478043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,2047,0.012455999851226807
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,2047,0.010979200154542923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,4095,0.012671999633312225
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,4095,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,8191,0.014731200039386749
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,8191,0.01465120017528534
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,16383,0.01698399931192398
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,16383,0.01664479970932007
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,1,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,1,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,3,0.011135999858379365
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,7,0.01101280003786087
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,7,0.010644800215959548
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,15,0.011700800061225891
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,15,0.010115200281143188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,31,0.010812799632549285
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,31,0.010835199803113937
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,63,0.01077599972486496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,63,0.01075040027499199
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,127,0.010903999954462052
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,127,0.010718400031328202
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,255,0.010923200100660325
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,255,0.010737600177526474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,3,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,511,0.010470400005578995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,511,0.01043360009789467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,1023,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,1023,0.010465600341558457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,2047,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,2047,0.010686399787664414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,4095,0.012615999579429627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,4095,0.012481600046157837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,8191,0.014748799800872802
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,8191,0.01252640038728714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,16383,0.016599999368190767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,1,0.008479999750852585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,1,0.008406399935483932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,3,0.008478400111198426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,3,0.008417599648237229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,7,0.008451200276613235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,7,0.008427199721336365
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,15,0.008495999872684479
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,15,0.008550400286912918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,31,0.008449599891901017
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,31,0.008460800349712371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,63,0.008470399677753449
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,63,0.008497600257396699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,127,0.008489599823951722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,511,0.01063840016722679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,127,0.008459199965000153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,255,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,255,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,511,0.008396799862384795
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,511,0.008452799916267396
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,1023,0.009798400104045868
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,2047,0.010644800215959548
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,2047,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,16383,0.029252800345420837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,4095,0.012572799623012543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,4095,0.010684800148010255
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,8191,0.027654400467872618
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,8191,0.014494399726390838
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,16383,0.04530239999294281
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,1,0.008513599634170532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,1,0.008550400286912918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,3,0.008568000048398972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,3,0.008620800077915191
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,7,0.008532799780368805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,7,0.008542399853467941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,15,0.01040479987859726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,15,0.008472000062465668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,31,0.010435199737548828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,255,0.008849599957466125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,31,0.00860000029206276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,63,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,63,0.00857279971241951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,127,0.01043360009789467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,127,0.008473599702119828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,255,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,255,0.008654399961233138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,511,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,511,0.008528000116348267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,1023,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,1023,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,2047,0.012583999335765839
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,2047,0.010665600001811982
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,16383,0.029009601473808287
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,4095,0.014567999541759491
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,8191,0.045475199818611145
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,8191,0.02882080078125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,16383,0.07980480194091796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,16383,0.045577600598335266
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,1,0.022793599963188173
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,1,0.018907199800014495
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,3,0.022899200022220612
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,1023,0.010251200199127198
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,3,0.018934400379657747
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,7,0.0227183997631073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,15,0.02272319942712784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,15,0.018724800646305086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,31,0.02272319942712784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,31,0.018721599876880646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,4095,0.027625599503517152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,63,0.01881600022315979
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,127,0.022888000309467315
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,127,0.01892800033092499
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,255,0.028977599740028382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,255,0.01886080056428909
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,511,0.04763680100440979
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,511,0.030932798981666565
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,1,0.02733280062675476
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,1,0.025076800584793092
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,7,0.018806399405002595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,3,0.02710399925708771
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,3,0.025051200389862062
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,7,0.024884800612926482
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,7,0.027305600047111512
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,15,0.0269679993391037
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,15,0.025011199712753295
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,31,0.024849599599838255
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,31,0.027323201298713684
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,63,0.026953598856925963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,63,0.024940800666809083
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,63,0.02274080067873001
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,127,0.030972799658775328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,255,0.04745439887046814
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,255,0.030976000428199767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,127,0.02483839988708496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,511,0.08181759715080261
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,511,0.048056000471115114
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,1,0.039208000898361205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,3,0.045368000864982605
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,3,0.03911199867725372
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,7,0.04535680115222931
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,7,0.03919839859008789
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,15,0.04536480009555817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,15,0.039124798774719236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,31,0.039094400405883786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,63,0.047486400604248045
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,63,0.03917120099067688
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,127,0.051209598779678345
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,127,0.041387200355529785
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,255,0.0823472023010254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,255,0.049744001030921935
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,511,0.1519984006881714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,1,0.0822704017162323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,1,0.06983199715614319
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,3,0.08224480152130127
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,3,0.06993439793586731
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,7,0.08236960172653199
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,7,0.07011520266532897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,1,0.045296001434326175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,15,0.08445600271224976
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,15,0.06795520186424256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,31,0.08467519879341126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,31,0.06988639831542968
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,31,0.04551360011100769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,63,0.08448960185050965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,63,0.06996960043907166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,127,0.09060959815979004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,127,0.07336959838867188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,255,0.15213760137557983
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,255,0.09041280150413514
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,1,0.03707360029220581
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,511,0.29053919315338134
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,1,0.028971201181411742
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,511,0.08408799767494202
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,3,0.03705280125141144
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,3,0.028988799452781676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,7,0.03722400069236755
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,15,0.03718239963054657
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,15,0.02902719974517822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,31,0.037190398573875426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,31,0.029009601473808287
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,63,0.037329599261283875
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,63,0.0291375994682312
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,127,0.03918400108814239
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,127,0.029183998703956604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,255,0.04853439927101135
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,255,0.033646398782730104
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,1,0.04539520144462585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,1,0.04130719900131226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,3,0.04121119976043701
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,7,0.045516800880432126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,511,0.1540976047515869
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,7,0.03915199935436249
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,15,0.045335999131202696
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,7,0.029020801186561584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,15,0.03927519917488098
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,31,0.04736959934234619
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,31,0.03923200070858002
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,63,0.03930560052394867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,127,0.05140320062637329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,127,0.043463999032974245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,255,0.08286399841308593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,3,0.04532800018787384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,255,0.05173599720001221
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,1,0.08240320086479187
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,1,0.07002400159835816
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,3,0.08233919739723206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,3,0.06986879706382751
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,7,0.070033597946167
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,15,0.0830735981464386
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,15,0.0699072003364563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,31,0.08357279896736144
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,31,0.06991360187530518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,63,0.08411999940872192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,63,0.0719327986240387
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,63,0.047758400440216064
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,127,0.09189119935035706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,127,0.07444159984588623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,255,0.15499199628829957
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,255,0.09086400270462036
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,1,0.15470399856567382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,1,0.1314576029777527
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,3,0.15438560247421265
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,3,0.13006399869918822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,7,0.12924480438232422
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,15,0.15673760175704957
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,7,0.08236960172653199
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,15,0.12935199737548828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,31,0.15823839902877807
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,31,0.13135039806365967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,63,0.13257919549942015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,127,0.17004799842834473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,127,0.13583040237426758
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,255,0.29484000205993655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,255,0.17377920150756837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,1,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,3,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,3,0.010526400059461594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,7,0.010542400181293488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,7,0.010569600015878677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,15,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,15,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,31,0.010567999631166457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,31,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,7,0.15834399461746215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,63,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,63,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,127,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,63,0.15763200521469117
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,255,0.01043199971318245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,511,0.01048479974269867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,511,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,1023,0.010444799810647965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,1023,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,2047,0.01250080019235611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,2047,0.012459199875593185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,4095,0.014537599682807923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,4095,0.012520000338554382
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,1,0.010545600205659866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,8191,0.016595199704170227
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,8191,0.014564800262451171
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,16383,0.030926400423049928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,16383,0.01844000071287155
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,1,0.008584000170230865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,3,0.008553600311279297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,3,0.008568000048398972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,7,0.008761599659919739
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,7,0.008529599756002426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,15,0.008607999980449676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,127,0.010585600137710571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,15,0.008665599673986436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,31,0.008529599756002426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,31,0.008628799766302108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,255,0.01061599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,63,0.00859839990735054
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,127,0.008630400151014328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,127,0.009611199796199798
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,255,0.008488000184297562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,255,0.008609600365161896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,511,0.008720000088214875
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,1023,0.010393600165843963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,1023,0.010907199978828431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,2047,0.010694400221109391
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,2047,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,4095,0.012671999633312225
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,4095,0.010632000118494033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,8191,0.027544000744819643
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,8191,0.014646400511264802
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,16383,0.04529919922351837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,16383,0.02881920039653778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,1,0.010420800000429154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,63,0.008628799766302108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,1,0.008603200316429138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,3,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,511,0.008620800077915191
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,3,0.00859680026769638
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,7,0.010409600287675857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,7,0.00857120007276535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,15,0.010704000294208527
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,15,0.008575999736785888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,31,0.008673600107431411
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,63,0.010628800094127654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,63,0.008670400083065032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,1,0.008515200018882752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,127,0.010835199803113937
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,127,0.008798400312662125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,255,0.00857279971241951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,255,0.008851200342178345
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,511,0.008551999926567078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,1023,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,1023,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,2047,0.012582400441169738
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,2047,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,4095,0.02754719853401184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,4095,0.012723200023174286
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,8191,0.04535839855670929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,8191,0.028804799914360045
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,31,0.010422399640083313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,16383,0.07998560070991516
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,16383,0.04592959880828858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,1,0.008627200126647949
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,3,0.009185600280761718
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,3,0.00852160006761551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,7,0.008537600189447403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,7,0.009105599671602248
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,15,0.008644799888134002
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,15,0.009078399837017059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,31,0.009028799831867218
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,63,0.008636800199747085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,31,0.008924800157546996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,63,0.008575999736785888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,127,0.00899839997291565
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,127,0.0086496002972126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,255,0.008739200234413148
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,255,0.008817599713802337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,511,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,511,0.009414400160312652
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,1023,0.012492799758911132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,2047,0.027327999472618103
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,1023,0.010993599891662598
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,2047,0.01273919939994812
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,4095,0.0457392007112503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,4095,0.027561599016189577
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,8191,0.07987679839134217
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,8191,0.045710399746894836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,16383,0.08066239953041077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,16383,0.1482751965522766
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,1,0.010372799634933472
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,1,0.008609600365161896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,1,0.01072160005569458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,3,0.008515200018882752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,7,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,15,0.010632000118494033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,15,0.010569600015878677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,511,0.008804800361394883
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,31,0.00878399983048439
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,63,0.009028799831867218
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,63,0.008636800199747085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,127,0.04853279888629913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,127,0.008528000116348267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,255,0.010390400141477584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,255,0.009196799993515015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,511,0.010444799810647965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,511,0.009352000057697296
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,1023,0.010564800351858139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,1023,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,2047,0.01250080019235611
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,2047,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,4095,0.02816160023212433
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,3,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,4095,0.014606399834156037
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,7,0.008563199639320373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,8191,0.02884959876537323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,16383,0.07906720042228699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,16383,0.0457152009010315
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,32767,0.14738719463348388
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,32767,0.08009120225906372
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,1,0.008499199897050858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,1,0.008448000252246856
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,3,0.008544000238180161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,3,0.008462399989366532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,7,0.008428800106048583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,7,0.008460800349712371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,15,0.008544000238180161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,15,0.0084927998483181
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,31,0.008470399677753449
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,31,0.008396799862384795
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,63,0.00857279971241951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,63,0.008484800159931184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,127,0.008513599634170532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,127,0.008455999940633774
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,255,0.008603200316429138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,255,0.008510400354862214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,511,0.010592000186443329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,511,0.00859680026769638
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,1023,0.012600000202655792
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,1023,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,2047,0.027024000883102417
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,8191,0.045419201254844666
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,2047,0.012729600071907043
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,4095,0.045286399126052854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,8191,0.07856000065803528
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,8191,0.045577600598335266
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,16383,0.1475167989730835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,16383,0.08055679798126221
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,32767,0.2838207960128784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,32767,0.15011839866638182
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,1,0.010998400300741196
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,1,0.01074879989027977
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,3,0.010619200021028518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,4095,0.027084800601005554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,3,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,7,0.010691200196743012
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,7,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,31,0.010644800215959548
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,15,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,15,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,31,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,31,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,63,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,63,0.010595200210809707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,127,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,127,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,255,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,255,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,511,0.012721599638462066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,511,0.010679999738931656
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,1023,0.027470400929450987
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,2047,0.045640000700950624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,4095,0.0804207980632782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,2047,0.028784000873565675
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,8191,0.1499008059501648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,8191,0.08202559947967529
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,16383,0.28866240978240965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,32767,0.5646848201751709
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,32767,0.2948240041732788
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,1,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,1023,0.014502400159835815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,1,0.008560000360012055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,3,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,3,0.008708799630403519
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,4095,0.04593920111656189
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,7,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,7,0.008673600107431411
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,15,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,31,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,31,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,63,0.010398399829864503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,63,0.010416000336408614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,127,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,127,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,255,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,255,0.010396800190210342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,511,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,511,0.010606399923563003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,1023,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,1023,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,2047,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,2047,0.010409600287675857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,4095,0.010664000362157821
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,4095,0.010681600123643876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,8191,0.012651200592517852
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,8191,0.01265919953584671
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,16383,0.01671359986066818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,16383,0.016672000288963318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,32767,0.016732800006866454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,1,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,1,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,3,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,3,0.008585599809885025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,7,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,7,0.008472000062465668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,15,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,15,0.008633600175380706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,15,0.008515200018882752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,31,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,31,0.008454400300979614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,63,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,63,0.00891520008444786
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,127,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,127,0.0090768001973629
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,255,0.010465600341558457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,255,0.00846719965338707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,16383,0.15406559705734252
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,511,0.008603200316429138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,1023,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,1023,0.010636799782514573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,2047,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,2047,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,4095,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,4095,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,8191,0.012548799812793731
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,8191,0.012600000202655792
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,32767,0.01857759952545166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,16383,0.01671999990940094
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,16383,0.018694399297237395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,32767,0.01952800005674362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,1,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,1,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,3,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,3,0.010391999781131745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,7,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,7,0.010228800028562546
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,15,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,15,0.009918399900197983
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,31,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,31,0.010040000081062317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,63,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,63,0.010420800000429154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,127,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,127,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,32767,0.01884479969739914
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,255,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,255,0.009809599816799163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,511,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,511,0.009723199903964997
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,1023,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,1023,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,2047,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,2047,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,4095,0.012567999958992004
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,4095,0.010603199899196624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,8191,0.014790399372577668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,8191,0.01406559944152832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,16383,0.016816000640392303
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,16383,0.014662399888038635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,32767,0.018916800618171692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,32767,0.016771200299263
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,1,0.008614400029182434
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,511,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,3,0.010500799864530563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,7,0.00857120007276535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,7,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,15,0.008564800024032593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,31,0.008575999736785888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,31,0.008542399853467941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,63,0.008550400286912918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,63,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,127,0.008504000306129456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,127,0.008548799902200699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,255,0.00857120007276535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,255,0.00857120007276535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,511,0.008979199826717377
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,511,0.008553600311279297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,1023,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,1,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,1023,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,2047,0.01043199971318245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,2047,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,4095,0.011208000034093857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,3,0.00931359976530075
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,8191,0.014715200662612915
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,8191,0.01257600039243698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,16383,0.01664319932460785
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,15,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,32767,0.018719999492168425
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,32767,0.018723200261592864
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,1,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,1,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,3,0.01043360009789467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,3,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,7,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,7,0.00857279971241951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,15,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,15,0.008515200018882752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,31,0.010422399640083313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,31,0.00865119993686676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,63,0.008523199707269669
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,63,0.008526399731636047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,127,0.008558399975299835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,127,0.00862559974193573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,255,0.009699200093746186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,255,0.008635199815034866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,511,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,511,0.009900800138711929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,1023,0.010571199655532836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,1023,0.010356800258159637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,2047,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,2047,0.010657600313425063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,4095,0.010630399733781815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,4095,0.012617599964141846
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,8191,0.014616000652313232
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,8191,0.014665600657463074
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,16383,0.01664000004529953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,16383,0.01669439971446991
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,32767,0.016739200055599212
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,16383,0.01870400011539459
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,1,0.010564800351858139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,1,0.01040479987859726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,3,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,3,0.010361599922180175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,7,0.01040479987859726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,7,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,15,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,15,0.0104032002389431
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,31,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,31,0.010454399883747101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,63,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,63,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,127,0.010569600015878677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,127,0.009851200133562088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,255,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,255,0.009855999797582626
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,4095,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,511,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,1023,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,1023,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,2047,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,2047,0.010619200021028518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,4095,0.012870399653911591
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,4095,0.012620800733566284
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,8191,0.014679999649524688
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,8191,0.01363680064678192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,16383,0.016732800006866454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,16383,0.014892800152301789
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,32767,0.03165439963340759
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,32767,0.016836799681186676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,1,0.008816000074148178
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,32767,0.01871040016412735
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,1,0.008638399839401244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,511,0.00921280011534691
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,3,0.008507200330495835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,3,0.008590400218963623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,7,0.008529599756002426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,7,0.008532799780368805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,31,0.00859680026769638
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,63,0.008564800024032593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,63,0.00838399976491928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,127,0.008507200330495835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,127,0.008560000360012055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,255,0.008894400298595428
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,255,0.008703999966382981
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,511,0.00902239978313446
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,511,0.008662399649620057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,1023,0.010251200199127198
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,1023,0.008828800171613693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,2047,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,2047,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,4095,0.012563200294971466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,4095,0.011215999722480774
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,8191,0.027832001447677612
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,8191,0.014619199931621552
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,16383,0.04557600021362305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,16383,0.02797600030899048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,32767,0.07973920106887818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,15,0.008638399839401244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,31,0.00849440023303032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,1,0.01029599979519844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,1,0.008444800227880477
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,3,0.008575999736785888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,7,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,7,0.008551999926567078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,15,0.008630400151014328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,15,0.00846560001373291
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,31,0.008782400190830231
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,31,0.009841600060462951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,63,0.010468800365924836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,63,0.00923840031027794
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,127,0.010739199817180634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,127,0.008585599809885025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,255,0.008497600257396699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,255,0.00854720026254654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,511,0.009875199943780898
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,511,0.008561599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,15,0.008542399853467941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,1023,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,2047,0.012668800354003907
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,2047,0.0122079998254776
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,4095,0.0287200003862381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,4095,0.014496000111103058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,8191,0.04546880125999451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,8191,0.028391999006271363
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,16383,0.07988640069961547
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,16383,0.045505601167678836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,32767,0.14741439819335939
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,32767,0.08024160265922546
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,1,0.008540800213813782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,1,0.008515200018882752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,3,0.00862400010228157
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,3,0.008544000238180161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,1023,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,7,0.008752000331878663
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,32767,0.04533120095729828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,7,0.008392000198364257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,15,0.008529599756002426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,3,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,31,0.00843999981880188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,63,0.008643200248479843
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,127,0.008483199775218964
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,127,0.00851999968290329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,255,0.00862239971756935
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,15,0.00846560001373291
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,31,0.008540800213813782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,255,0.008553600311279297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,63,0.008577600121498108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,511,0.00856959968805313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,1023,0.01247519999742508
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,1023,0.010659199953079224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,2047,0.027060800790786745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,2047,0.012652799487113953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,4095,0.04536640048027039
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,4095,0.027164798974990845
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,8191,0.045403200387954715
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,8191,0.07997120022773743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,16383,0.08066080212593078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,16383,0.1483504056930542
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,32767,0.14957280158996583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,32767,0.2850752115249634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,1,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,1,0.010582400113344192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,3,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,7,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,7,0.008852799981832504
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,15,0.010471999645233154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,15,0.008766400068998337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,31,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,31,0.009348800033330917
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,63,0.010435199737548828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,63,0.008558399975299835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,127,0.010564800351858139
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,127,0.008609600365161896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,255,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,255,0.008579199761152267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,511,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,511,0.008643200248479843
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,1023,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,1023,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,2047,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,2047,0.0106175996363163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,4095,0.01050880029797554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,4095,0.011844799667596818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,8191,0.014569599926471711
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,8191,0.014532800018787383
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,16383,0.016622400283813475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,16383,0.014638400077819825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,32767,0.018750399351119995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,32767,0.016625599563121797
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,1,0.00995199978351593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,1,0.008515200018882752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,3,0.010545600205659866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,3,0.008497600257396699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,7,0.010550399869680404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,3,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,7,0.008479999750852585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,15,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,15,0.010304000228643417
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,31,0.008729600161314011
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,31,0.010265599936246872
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,63,0.00852160006761551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,127,0.008535999804735184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,255,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,255,0.008643200248479843
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,511,0.01058719977736473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,511,0.010571199655532836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,1023,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,1023,0.010545600205659866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,2047,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,2047,0.010667199641466141
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,4095,0.012913599610328674
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,4095,0.012651200592517852
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,8191,0.014708800613880158
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,8191,0.012768000364303589
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,16383,0.01672320067882538
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,16383,0.014591999351978302
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,32767,0.031140801310539246
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,63,0.010416000336408614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,32767,0.016678400337696075
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,1,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,1,0.008662399649620057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,3,0.008867199718952178
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,7,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,7,0.008568000048398972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,15,0.010556799918413162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,15,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,31,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,31,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,63,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,63,0.008814399689435959
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,127,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,127,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,255,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,255,0.010555200278759003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,511,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,511,0.009239999949932099
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,1023,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,1023,0.01058719977736473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,127,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,3,0.010367999970912933
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,2047,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,4095,0.012990400195121765
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,8191,0.014864000678062438
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,2047,0.01064160019159317
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,8191,0.012639999389648438
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,4095,0.012587200105190276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,16383,0.029947200417518617
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,32767,0.04746400117874146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,32767,0.030235201120376587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,1,0.008484800159931184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,1,0.008551999926567078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,3,0.008551999926567078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,3,0.008551999926567078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,7,0.00857279971241951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,7,0.008495999872684479
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,15,0.008542399853467941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,15,0.008556800335645676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,31,0.008516799658536911
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,31,0.008483199775218964
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,511,0.010673599690198899
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,63,0.008561599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,127,0.008414400368928909
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,16383,0.015513600409030914
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,255,0.008544000238180161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,255,0.008486399799585343
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,511,0.010582400113344192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,511,0.010639999806880952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,1023,0.012675200402736665
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,1023,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,2047,0.02696160078048706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,2047,0.012724800407886505
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,4095,0.04431520104408264
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,4095,0.028300800919532777
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,8191,0.07867199778556824
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,8191,0.045342400670051575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,16383,0.14765119552612305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,16383,0.08027520179748535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,1,0.010654400289058685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,1,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,3,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,3,0.009614399820566177
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,7,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,7,0.010356800258159637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,15,0.010636799782514573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,15,0.010367999970912933
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,31,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,31,0.01040479987859726
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,63,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,63,0.010339199751615524
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,127,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,127,0.010652799904346467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,63,0.00840959995985031
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,255,0.010579200088977813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,255,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,127,0.008611200004816055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,511,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,1023,0.02685759961605072
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,1023,0.014628799259662628
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,2047,0.044865599274635314
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,2047,0.02869440019130707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,4095,0.07966560125350952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,4095,0.0455375999212265
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,8191,0.14818880558013917
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,16383,0.2856496095657349
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,16383,0.15348800420761108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,1,0.012596799433231354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,1,0.012603199481964112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,3,0.012579199671745301
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,3,0.012459199875593185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,7,0.012577599287033081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,7,0.012535999715328216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,15,0.012521600723266602
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,31,0.012614400684833526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,31,0.012537600100040435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,63,0.012615999579429627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,63,0.01263200044631958
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,127,0.012544000148773193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,127,0.012651200592517852
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,255,0.012644800543785095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,511,0.012683199346065521
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,255,0.012577599287033081
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,511,0.02847039997577667
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,511,0.014672000706195832
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,1023,0.04614399969577789
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,1023,0.028984001278877257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,2047,0.08177120089530945
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,2047,0.046932798624038694
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,4095,0.08195520043373108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,4095,0.15226080417633056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,8191,0.15265599489212037
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,8191,0.2940864086151123
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,15,0.012622399628162384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,1,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,1,0.010454399883747101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,16383,0.29418718814849854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,16383,0.5789552211761475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,3,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,7,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,7,0.01067039966583252
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,15,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,15,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,31,0.01058719977736473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,31,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,63,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,63,0.010555200278759003
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,127,0.010567999631166457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,127,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,255,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,8191,0.08174560070037842
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,255,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,511,0.012683199346065521
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,511,0.012283200025558471
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,1023,0.027476799488067628
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,1023,0.014614400267601014
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,2047,0.045377600193023684
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,2047,0.028190401196479798
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,4095,0.08051040172576904
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,4095,0.04594559967517853
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,1,0.012886400520801543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,1,0.012624000012874604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,3,0.012689599394798278
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,3,0.012593600153923034
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,3,0.01056319996714592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,7,0.012638400495052337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,15,0.012689599394798278
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,15,0.01268479973077774
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,31,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,63,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,63,0.012561599910259246
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,127,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,127,0.012604799866676331
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,255,0.012580800056457519
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,255,0.012334399670362473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,511,0.027239999175071715
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,511,0.014697599411010741
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,1023,0.04597600102424622
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,1023,0.02922239899635315
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,2047,0.08148000240325928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,7,0.012652799487113953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,2047,0.047470399737358095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,4095,0.0823472023010254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,31,0.012564800679683685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,1,0.016945600509643555
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,1,0.014753599464893342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,3,0.0182559996843338
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,3,0.014913600683212281
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,7,0.016791999340057373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,15,0.016764800250530242
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,7,0.016947199404239655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,31,0.01675360053777695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,31,0.015512000024318694
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,63,0.016704000532627106
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,127,0.016680000722408293
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,127,0.014596800506114959
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,255,0.02682720124721527
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,255,0.014686399698257446
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,511,0.045433598756790164
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,511,0.028681600093841554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,1023,0.07982879877090454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,1023,0.04640479981899261
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,2047,0.14805279970169066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,2047,0.08182240128517151
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,4095,0.15179840326309205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,4095,0.28581600189208983
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,4095,0.15236959457397461
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,15,0.014708800613880158
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,1,0.012535999715328216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,1,0.012539200484752655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,3,0.01255200058221817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,7,0.01268800050020218
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,7,0.012622399628162384
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,63,0.01520799994468689
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,15,0.01250240057706833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,31,0.012656000256538392
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,31,0.012495999783277511
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,63,0.012603199481964112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,63,0.012564800679683685
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,127,0.01281760036945343
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,127,0.012559999525547028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,255,0.01271200031042099
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,255,0.012470400333404541
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,511,0.02895520031452179
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,511,0.014752000570297241
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,1023,0.04678240120410919
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,1023,0.028992000222206115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,2047,0.08217759728431702
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,2047,0.04765599966049194
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,1,0.017115199565887453
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,1,0.014683200418949128
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,3,0.016945600509643555
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,3,0.014937600493431092
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,7,0.017372800409793852
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,7,0.014665600657463074
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,15,0.016814400255680085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,15,0.014876799285411834
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,31,0.016832000017166136
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,31,0.016663999855518342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,63,0.017023999989032746
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,63,0.016624000668525696
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,127,0.016761599481105803
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,127,0.016654400527477263
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,3,0.012556800246238708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,255,0.028003200888633728
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,255,0.016715200245380403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,15,0.012574400007724761
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,511,0.029150399565696716
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,1023,0.08027039766311646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,1023,0.0475055992603302
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,2047,0.14815839529037475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,2047,0.08281599879264831
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,1,0.027112001180648805
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,1,0.02306399941444397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,3,0.02699039876461029
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,3,0.022886399924755097
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,7,0.026977598667144775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,7,0.02298559993505478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,15,0.027025601267814635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,15,0.022860799729824067
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,31,0.027008000016212463
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,31,0.02296479940414429
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,63,0.027036800980567932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,63,0.022801600396633148
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,127,0.02288320064544678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,255,0.04551199972629547
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,255,0.030004799365997314
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,511,0.04532960057258606
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,511,0.08049920201301575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,511,0.046744000911712644
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,1023,0.1492735981941223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,1023,0.08180959820747376
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,2047,0.15062400102615356
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,2047,0.2867376089096069
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,1,0.008776000142097473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,1,0.00849440023303032
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,3,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,3,0.008539199829101562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,7,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,7,0.008531200140714646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,15,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,15,0.008604799956083297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,31,0.010780800133943558
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,63,0.009118399769067764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,31,0.010782399773597717
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,127,0.030318400263786315
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,127,0.010751999914646149
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,255,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,511,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,511,0.009921599924564362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,1023,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,1023,0.010619200021028518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,2047,0.010620799660682679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,2047,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,4095,0.012619200348854064
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,4095,0.012537600100040435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,8191,0.014620800316333771
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,8191,0.012727999687194824
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,16383,0.016803200542926788
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,16383,0.014844800531864166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,32767,0.031014400720596313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,63,0.008449599891901017
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,127,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,1,0.01043360009789467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,255,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,3,0.010065600275993347
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,3,0.010366400331258773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,7,0.010001599788665771
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,7,0.010407999902963639
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,15,0.009614399820566177
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,15,0.010390400141477584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,31,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,31,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,63,0.010361599922180175
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,63,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,127,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,255,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,127,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,511,0.010542400181293488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,511,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,1023,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,1023,0.010440000146627427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,2047,0.01191679984331131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,2047,0.010665600001811982
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,4095,0.012745599448680877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,4095,0.012544000148773193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,8191,0.014691199362277984
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,32767,0.016598400473594666
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,8191,0.012697599828243256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,16383,0.029655998945236205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,1,0.009495999664068222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,16383,0.016568000614643096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,32767,0.04752480089664459
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,1,0.008577600121498108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,3,0.009888000041246413
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,3,0.008614400029182434
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,7,0.009473600238561631
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,7,0.008628799766302108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,15,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,255,0.010440000146627427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,15,0.00854720026254654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,31,0.008515200018882752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,63,0.008513599634170532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,31,0.008579199761152267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,63,0.008462399989366532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,1,0.009759999811649323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,127,0.009756799787282944
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,255,0.009804800152778625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,255,0.008627200126647949
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,511,0.008563199639320373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,511,0.008772800117731095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,1023,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,1023,0.008696000277996063
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,127,0.008849599957466125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,2047,0.010665600001811982
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,2047,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,4095,0.010756800323724747
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,8191,0.027193599939346315
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,8191,0.014668799936771393
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,16383,0.04528799951076508
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,16383,0.028355199098587035
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,32767,0.030449599027633667
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,32767,0.0798479974269867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,32767,0.045665600895881654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,1,0.01836639940738678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,1,0.016641600430011748
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,3,0.016707199811935424
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,7,0.01690559983253479
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,7,0.016711999475955964
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,15,0.01677599996328354
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,15,0.016678400337696075
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,31,0.01677920073270798
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,4095,0.012603199481964112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,63,0.017206400632858276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,63,0.016680000722408293
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,127,0.016864000260829924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,127,0.016832000017166136
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,255,0.027663999795913698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,255,0.016758400201797485
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,511,0.045326399803161624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,511,0.02943840026855469
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,1023,0.08026720285415649
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,1023,0.04771519899368286
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,1,0.026924800872802735
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,1,0.024907200038433074
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,3,0.026903998851776124
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,3,0.024736000597476958
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,7,0.026976001262664796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,7,0.024879999458789825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,15,0.027059200406074523
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,15,0.02473919987678528
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,3,0.018617600202560425
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,63,0.027055999636650084
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,63,0.0247871994972229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,127,0.030910399556159974
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,127,0.024830399453639983
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,31,0.016732800006866454
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,255,0.04579519927501678
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,255,0.03078399896621704
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,511,0.08086079955101014
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,1023,0.1491760015487671
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,1023,0.08303999900817871
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,1,0.0454367995262146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,1,0.039294400811195375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,3,0.04545280039310455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,3,0.039340800046920775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,7,0.04545759856700897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,31,0.02473440021276474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,7,0.039259201288223265
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,15,0.045521599054336545
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,15,0.03923520147800445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,31,0.04541600048542023
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,31,0.039550399780273436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,63,0.047495999932289125
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,63,0.03925279974937439
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,127,0.05102239847183228
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,127,0.04158720076084137
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,255,0.08161759972572327
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,255,0.049451199173927304
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,511,0.047383999824523924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,511,0.1499359965324402
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,511,0.08252800107002259
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,1,0.02696160078048706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,1023,0.28722400665283204
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,1,0.025036799907684325
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,3,0.026894399523735048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,7,0.026958400011062623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,7,0.02494720071554184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,15,0.026918399333953857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,15,0.024928000569343568
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,31,0.02696160078048706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,31,0.026931199431419372
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,31,0.024864000082015992
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,63,0.02696320116519928
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,63,0.024953599274158477
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,127,0.031001600623130798
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,127,0.024928000569343568
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,255,0.04680800139904022
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,255,0.031491199135780336
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,511,0.08195359706878662
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,511,0.04919680058956146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,1,0.04532159864902496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,3,0.045332801342010495
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,3,0.03922240138053894
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,7,0.04535360038280487
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,1023,0.15253920555114747
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,7,0.0394896000623703
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,15,0.0454479992389679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,15,0.03995679914951324
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,31,0.046112000942230225
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,31,0.039603200554847715
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,63,0.04773440062999725
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,63,0.03940320014953613
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,127,0.05121279954910278
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,127,0.04367679953575134
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,1,0.03922080099582672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,255,0.08235999941825867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,255,0.050862401723861694
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,511,0.1517583966255188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,511,0.0844543993473053
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,1,0.08225119709968567
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,1,0.07009440064430236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,3,0.024987199902534486
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,3,0.07003679871559143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,7,0.08233439922332764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,7,0.0703440010547638
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,15,0.08428800106048584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,15,0.07012959718704223
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,31,0.08462079763412475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,31,0.07012479901313781
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,63,0.08462079763412475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,63,0.07184799909591674
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,127,0.091212797164917
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,127,0.07401440143585206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,255,0.1527791976928711
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,255,0.08899199962615967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,1,0.009459199756383896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,511,0.15447200536727906
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,1,0.008526399731636047
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,511,0.2904720067977905
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,3,0.010387200117111205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,3,0.008556800335645676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,7,0.010486400127410889
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,15,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,15,0.00856959968805313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,31,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,31,0.009387200325727462
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,63,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,63,0.008531200140714646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,127,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,127,0.010147199779748917
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,255,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,255,0.010123199969530105
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,511,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,511,0.008556800335645676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,1023,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,1023,0.01019359976053238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,2047,0.010569600015878677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,2047,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,4095,0.012744000554084778
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,4095,0.012505599856376648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,8191,0.014720000326633453
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,8191,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,16383,0.029348799586296083
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,16383,0.015568000078201295
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,32767,0.0471343994140625
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,32767,0.02921920120716095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,1,0.008511999994516373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,1,0.008460800349712371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,3,0.008588799834251403
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,3,0.00857120007276535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,7,0.00851840004324913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,7,0.008582399785518646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,15,0.008529599756002426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,15,0.008628799766302108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,31,0.008539199829101562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,31,0.008473599702119828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,63,0.008452799916267396
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,63,0.008491200208663941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,127,0.008540800213813782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,7,0.01005759984254837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,255,0.00852160006761551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,255,0.008595199882984161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,511,0.008486399799585343
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,511,0.008591999858617782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,1023,0.01061599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,1023,0.009164799749851228
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,2047,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,3,0.0823087990283966
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,4095,0.012455999851226807
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,8191,0.027694401144981385
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,8191,0.014553600549697876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,16383,0.04536640048027039
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,16383,0.028833600878715514
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,32767,0.07931519746780395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,32767,0.0456928014755249
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,1,0.010105600208044052
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,1,0.008502399921417237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,3,0.010238400101661682
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,3,0.008595199882984161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,7,0.010308799892663955
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,7,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,127,0.008478400111198426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,15,0.010387200117111205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,15,0.008743999898433686
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,31,0.01032159999012947
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,31,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,63,0.010465600341558457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,2047,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,63,0.010401599854230881
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,127,0.010344000160694122
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,127,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,255,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,511,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,511,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,1023,0.010659199953079224
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,1023,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,2047,0.012651200592517852
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,2047,0.010673599690198899
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,4095,0.02773120105266571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,4095,0.014633600413799287
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,8191,0.04538559913635254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,8191,0.028913599252700806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,16383,0.08046560287475586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,4095,0.012644800543785095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,32767,0.14822399616241455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,255,0.008563199639320373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,1,0.010632000118494033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,1,0.008635199815034866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,3,0.010364799946546554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,3,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,7,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,15,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,7,0.00902400016784668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,15,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,31,0.00931359976530075
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,63,0.009204799681901932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,63,0.008934400230646133
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,127,0.009839999675750732
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,127,0.010702399909496308
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,255,0.01072160005569458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,16383,0.04567199945449829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,511,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,511,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,1023,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,32767,0.08015999794006348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,1023,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,2047,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,4095,0.027857598662376405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,4095,0.01470080018043518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,8191,0.045284798741340636
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,8191,0.028547200560569762
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,16383,0.07888320088386536
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,16383,0.04551360011100769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,32767,0.14761439561843873
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,32767,0.07995520234107971
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,65535,0.28442559242248533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,255,0.009991999715566635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,1,0.008859200030565261
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,65535,0.14805760383605956
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,1,0.008668799698352814
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,3,0.008544000238180161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,3,0.008615999668836593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,2047,0.010630399733781815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,7,0.008491200208663941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,15,0.008630400151014328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,15,0.008511999994516373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,7,0.008628799766302108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,31,0.008603200316429138
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,31,0.008462399989366532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,63,0.008782400190830231
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,127,0.008542399853467941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,255,0.008528000116348267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,127,0.0086496002972126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,255,0.008587200194597244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,511,0.010649599879980088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,31,0.009003199636936188
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,1023,0.012652799487113953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,2047,0.027036800980567932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,2047,0.012830400466918945
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,4095,0.0452208012342453
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,4095,0.02779200077056885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,8191,0.07870720028877258
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,8191,0.0454367995262146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,16383,0.14714399576187134
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,16383,0.07991999983787537
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,63,0.008566399663686752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,32767,0.28391199111938475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,32767,0.14885760545730592
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,511,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,1023,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,1,0.00857279971241951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,65535,0.5563568115234375
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,65535,0.28781120777130126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,3,0.008449599891901017
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,7,0.008459199965000153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,15,0.009537599980831146
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,15,0.008483199775218964
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,31,0.009361600130796432
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,31,0.008550400286912918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,63,0.010380800068378448
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,63,0.008472000062465668
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,127,0.008556800335645676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,127,0.008499199897050858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,255,0.008507200330495835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,1,0.008523199707269669
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,255,0.008580800145864487
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,3,0.010384000092744827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,511,0.008539199829101562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,511,0.008595199882984161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,1023,0.010521599650382995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,1023,0.008577600121498108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,7,0.01043360009789467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,2047,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,4095,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,4095,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,8191,0.012774400413036346
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,8191,0.01263200044631958
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,16383,0.016739200055599212
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,32767,0.018654400110244752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,32767,0.01679359972476959
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,65535,0.02003680020570755
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,65535,0.01865600049495697
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,1,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,1,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,3,0.011078400164842605
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,3,0.008609600365161896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,7,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,7,0.008606400340795517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,15,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,15,0.008556800335645676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,2047,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,31,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,31,0.00857120007276535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,63,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,63,0.008535999804735184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,127,0.010345599800348281
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,255,0.008627200126647949
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,127,0.010876800119876861
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,255,0.010353600233793258
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,511,0.00857279971241951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,1023,0.010393600165843963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,1023,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,2047,0.010556799918413162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,2047,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,4095,0.011579199880361556
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,4095,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,8191,0.014553600549697876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,8191,0.012556800246238708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,16383,0.01857759952545166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,32767,0.018798400461673737
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,32767,0.018638400733470915
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,65535,0.02083040028810501
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,65535,0.0205935999751091
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,1,0.010398399829864503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,1,0.010070399940013885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,3,0.010358399897813796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,3,0.009796799719333648
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,7,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,7,0.008507200330495835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,15,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,15,0.008534400165081025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,31,0.010468800365924836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,31,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,16383,0.016739200055599212
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,63,0.01027199998497963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,63,0.010435199737548828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,127,0.010369600355625152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,127,0.010452800244092942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,255,0.010409600287675857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,255,0.01029599979519844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,511,0.010396800190210342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,511,0.010369600355625152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,1023,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,1023,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,2047,0.010540799796581268
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,4095,0.011475200206041336
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,4095,0.01061599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,8191,0.012673600018024445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,8191,0.01281760036945343
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,16383,0.018649600446224213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,16383,0.018614399433135986
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,32767,0.01883520036935806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,32767,0.018751999735832213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,65535,0.02091519981622696
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,65535,0.01886720061302185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,1,0.01056160032749176
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,1,0.008558399975299835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,3,0.010465600341558457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,3,0.008601599931716919
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,511,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,7,0.010407999902963639
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,7,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,15,0.010116799920797347
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,2047,0.010711999982595444
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,31,0.010328000038862228
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,63,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,63,0.010396800190210342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,127,0.010283199697732925
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,255,0.010390400141477584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,255,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,511,0.010339199751615524
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,511,0.009758400171995163
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,1023,0.010375999659299851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,15,0.010398399829864503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,1023,0.01043199971318245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,31,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,2047,0.010595200210809707
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,16383,0.01674720048904419
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,2047,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,4095,0.012462399899959564
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,4095,0.010608000308275222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,8191,0.014571200311183929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,16383,0.016735999286174773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,16383,0.01664000004529953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,32767,0.018724800646305086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,32767,0.017057600617408752
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,65535,0.03498719930648804
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,65535,0.01890240013599396
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,1,0.008584000170230865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,1,0.008638399839401244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,3,0.008631999790668487
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,3,0.008633600175380706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,7,0.00856959968805313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,7,0.008548799902200699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,15,0.008700799942016602
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,31,0.008584000170230865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,31,0.008636800199747085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,63,0.00862400010228157
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,63,0.008664000034332275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,127,0.008763200044631958
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,127,0.009462399780750275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,255,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,8191,0.014460800588130951
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,511,0.00846880003809929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,511,0.008739200234413148
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,1023,0.009995199739933014
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,1023,0.008926399797201157
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,2047,0.010691200196743012
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,2047,0.010449600219726563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,4095,0.012585599720478059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,4095,0.01249919980764389
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,8191,0.02723200023174286
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,16383,0.04534080028533936
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,8191,0.014529600739479065
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,16383,0.028543999791145323
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,32767,0.07984319925308228
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,32767,0.045526400208473206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,65535,0.08060160279273987
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,65535,0.14761439561843873
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,1,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,3,0.00904960036277771
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,1,0.008620800077915191
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,3,0.008617600053548813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,7,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,7,0.008708799630403519
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,15,0.008716800063848496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,31,0.009548799693584442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,31,0.008944000303745269
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,63,0.009387200325727462
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,63,0.008958400040864945
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,127,0.01228640004992485
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,127,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,255,0.01064319983124733
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,255,0.010080000013113022
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,15,0.008568000048398972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,511,0.009756799787282944
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,511,0.008892799913883209
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,1023,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,1023,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,2047,0.012614400684833526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,4095,0.027484801411628724
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,4095,0.014657600224018097
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,8191,0.04540959894657135
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,8191,0.028935998678207397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,15,0.010363200306892395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,16383,0.07933440208435058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,16383,0.045694398880004886
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,32767,0.14742560386657716
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,32767,0.08025760054588318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,65535,0.1495695948600769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,1,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,1,0.010396800190210342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,3,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,3,0.010337600111961364
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,7,0.010390400141477584
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,7,0.008502399921417237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,2047,0.010708799958229065
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,15,0.010292799770832061
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,15,0.008799999952316284
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,31,0.010375999659299851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,31,0.008689600229263305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,63,0.010502400249242783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,63,0.008491200208663941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,127,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,127,0.009174399822950364
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,255,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,255,0.008534400165081025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,511,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,511,0.008550400286912918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,1023,0.01043199971318245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,1023,0.008705600351095199
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,2047,0.010630399733781815
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,255,0.008745600283145905
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,2047,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,4095,0.012452799826860428
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,4095,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,8191,0.014740799367427827
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,8191,0.012571200728416443
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,16383,0.016758400201797485
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,16383,0.01470080018043518
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,32767,0.0188511997461319
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,32767,0.016820800304412842
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,65535,0.03478240072727203
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,65535,0.01875839978456497
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,1,0.010454399883747101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,1,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,3,0.010382399708032609
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,3,0.010556799918413162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,7,0.010444799810647965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,7,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,15,0.010428799688816071
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,15,0.008524800091981888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,31,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,31,0.008694399893283845
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,63,0.008558399975299835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,127,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,127,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,255,0.010620799660682679
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,255,0.010632000118494033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,511,0.010532800108194351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,511,0.010552000254392624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,1023,0.01058719977736473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,1023,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,65535,0.2842191934585571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,2047,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,4095,0.012600000202655792
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,4095,0.012019199877977371
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,8191,0.014387199282646179
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,63,0.010398399829864503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,8191,0.012742400169372559
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,16383,0.016891199350357055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,16383,0.014737600088119506
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,32767,0.03175680041313171
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,32767,0.016475200653076172
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,65535,0.049384000897407535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,65535,0.03166399896144867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,1,0.008580800145864487
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,1,0.008635199815034866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,3,0.008593600243330002
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,3,0.00851999968290329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,7,0.008528000116348267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,7,0.008516799658536911
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,15,0.008568000048398972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,15,0.008476799726486206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,31,0.0086496002972126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,31,0.008615999668836593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,63,0.00854720026254654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,63,0.008657599985599517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,127,0.00854720026254654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,127,0.008502399921417237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,255,0.009617599844932555
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,255,0.008591999858617782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,2047,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,511,0.010655999928712846
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,1023,0.012608000636100769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,2047,0.02690880000591278
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,2047,0.012763200700283051
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,4095,0.04518559873104096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,8191,0.07871999740600585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,8191,0.04537439942359924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,16383,0.14653279781341552
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,16383,0.07895519733428955
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,32767,0.28389439582824705
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,32767,0.14828319549560548
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,1,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,1,0.01058719977736473
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,511,0.010739199817180634
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,3,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,1023,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,3,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,7,0.010596799850463866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,7,0.01051200032234192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,15,0.010639999806880952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,15,0.010468800365924836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,31,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,4095,0.028604799509048463
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,63,0.01040000021457672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,127,0.01053439974784851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,127,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,255,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,511,0.012654399871826172
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,511,0.010660800337791442
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,1023,0.027079999446868896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,1023,0.014486399292945863
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,2047,0.04533439874649048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,2047,0.028718400001525878
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,4095,0.08093439936637878
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,4095,0.0467056006193161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,8191,0.14817440509796143
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,8191,0.08235520124435425
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,31,0.010558400303125381
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,16383,0.2863296031951904
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,63,0.010552000254392624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,255,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,16383,0.15256479978561402
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,1,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,1,0.010665600001811982
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,32767,0.5630896091461182
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,32767,0.29670400619506837
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,7,0.010628800094127654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,7,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,15,0.010711999982595444
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,15,0.010473600029945374
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,31,0.010633599758148194
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,31,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,63,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,63,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,127,0.010553599894046783
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,127,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,255,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,255,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,511,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,3,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,1023,0.027459201216697694
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,3,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,1023,0.014595200121402741
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,2047,0.04536159932613373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,2047,0.028033599257469177
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,4095,0.07979999780654908
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,4095,0.04549440145492554
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,8191,0.14803999662399292
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,1,0.012611199915409089
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,1,0.012559999525547028
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,3,0.012747199833393097
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,3,0.01255200058221817
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,7,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,7,0.012860800325870513
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,15,0.013150399923324585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,31,0.01279039978981018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,31,0.012452799826860428
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,63,0.012676799297332763
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,511,0.012489599734544754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,63,0.01250240057706833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,127,0.012694400548934937
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,127,0.012572799623012543
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,255,0.01276639997959137
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,255,0.012851199507713318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,511,0.028497600555419923
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,511,0.014657600224018097
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,1023,0.04560000002384186
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,1023,0.029102399945259094
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,2047,0.04751519858837128
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,8191,0.0805903971195221
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,4095,0.15204479694366455
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,4095,0.08212320208549499
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,8191,0.15226880311965943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,1,0.012588800489902496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,1,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,3,0.01266240030527115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,3,0.012507200241088867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,15,0.012777599692344665
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,7,0.01257600039243698
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,7,0.012539200484752655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,15,0.012630400061607362
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,15,0.012556800246238708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,31,0.013121600449085235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,31,0.01252640038728714
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,63,0.012587200105190276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,63,0.012535999715328216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,127,0.012588800489902496
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,127,0.012563200294971466
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,255,0.012535999715328216
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,255,0.012582400441169738
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,511,0.01467680037021637
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,2047,0.0819760024547577
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,1023,0.04596799910068512
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,1023,0.029142400622367857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,2047,0.08195840120315552
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,2047,0.047156798839569095
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,4095,0.08228960037231445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,4095,0.1519711971282959
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,8191,0.2923295974731445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,1,0.016729600727558136
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,1,0.016708800196647645
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,3,0.016542400419712066
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,3,0.016806399822235106
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,7,0.016510400176048278
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,15,0.017956799268722533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,15,0.01655520051717758
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,31,0.018367999792099
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,31,0.01669279932975769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,63,0.018004800379276275
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,63,0.016627199947834015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,127,0.01825920045375824
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,127,0.01652639955282211
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,255,0.02717919945716858
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,255,0.016524800658226015
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,511,0.04539520144462585
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,511,0.028963199257850646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,1023,0.08004000186920165
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,1023,0.04677119851112366
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,2047,0.147980797290802
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,2047,0.08086079955101014
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,511,0.02831200063228607
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,4095,0.15284160375595093
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,1,0.010497599840164185
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,1,0.008595199882984161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,3,0.010582400113344192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,3,0.008692800253629684
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,7,0.01664000004529953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,7,0.008593600243330002
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,15,0.009865599870681762
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,15,0.008587200194597244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,31,0.010233599692583084
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,31,0.008636800199747085
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,63,0.00905120000243187
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,63,0.008668799698352814
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,127,0.010571199655532836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,127,0.008577600121498108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,4095,0.2842144012451172
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,255,0.010571199655532836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,255,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,511,0.01055999994277954
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,511,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,1023,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,1023,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,2047,0.010592000186443329
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,2047,0.010676799714565277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,7,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,4095,0.012539200484752655
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,8191,0.0147024005651474
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,8191,0.01276479959487915
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,16383,0.01674399971961975
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,16383,0.014803199470043183
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,32767,0.01669919937849045
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,65535,0.04911839962005615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,65535,0.03176479935646057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,1,0.010603199899196624
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,1,0.01043199971318245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,3,0.01040000021457672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,3,0.01053439974784851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,7,0.010436800122261048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,7,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,15,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,15,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,31,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,31,0.01043360009789467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,63,0.011028800159692764
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,63,0.010371199995279311
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,127,0.01037440001964569
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,127,0.010585600137710571
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,255,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,255,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,511,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,511,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,1023,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,1023,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,2047,0.011737599968910217
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,2047,0.010649599879980088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,4095,0.012734399735927581
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,4095,0.012470400333404541
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,8191,0.014742399752140044
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,8191,0.012600000202655792
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,16383,0.030364799499511718
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,16383,0.01672160029411316
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,32767,0.031118398904800414
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,32767,0.047414401173591615
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,32767,0.0308351993560791
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,65535,0.08091999888420105
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,1,0.016791999340057373
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,1,0.01669919937849045
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,3,0.016659200191497803
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,3,0.016707199811935424
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,7,0.016606399416923524
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,15,0.016680000722408293
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,15,0.016659200191497803
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,31,0.016784000396728515
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,31,0.016735999286174773
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,63,0.016678400337696075
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,63,0.016628800332546233
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,127,0.016918399930000307
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,127,0.01669600009918213
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,255,0.027718400955200194
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,4095,0.012625600397586822
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,255,0.016705599427223206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,511,0.02903839945793152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,511,0.04531520009040833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,1023,0.07934079766273498
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,1023,0.04724319875240326
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,2047,0.0820576012134552
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,65535,0.04899519979953766
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,1,0.02691200077533722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,1,0.02476480007171631
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,3,0.026924800872802735
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,3,0.024864000082015992
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,7,0.027051201462745665
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,15,0.026943999528884887
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,15,0.024852800369262695
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,31,0.02691200077533722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,31,0.02298240065574646
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,63,0.026953598856925963
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,63,0.022968000173568724
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,127,0.029552000761032104
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,2047,0.14784959554672242
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,127,0.02463040053844452
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,7,0.01863040030002594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,255,0.04540959894657135
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,7,0.024702399969100952
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,255,0.030881598591804504
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,511,0.08023999929428101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,511,0.04732640087604523
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,1023,0.08220000267028808
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,2047,0.28497920036315916
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,2047,0.15208319425582886
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,1,0.026859200000762938
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,1,0.025035199522972108
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,3,0.026899200677871705
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,7,0.026836800575256347
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,3,0.025088000297546386
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,7,0.025011199712753295
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,15,0.027188798785209654
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,31,0.027011200785636902
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,31,0.024876800179481507
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,63,0.027001601457595826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,63,0.0249439999461174
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,127,0.03084160089492798
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,127,0.024884800612926482
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,255,0.04583840072154999
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,255,0.031404799222946166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,1023,0.14847359657287598
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,511,0.0805903971195221
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,511,0.04913119971752167
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,1023,0.1496399998664856
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,1,0.04545120000839233
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,1,0.039566400647163394
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,3,0.0453792005777359
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,3,0.039320001006126405
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,7,0.04541600048542023
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,7,0.039211198687553406
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,15,0.04540959894657135
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,15,0.03922559916973114
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,31,0.04606879949569702
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,31,0.03929280042648316
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,63,0.047512000799179076
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,63,0.0392655998468399
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,127,0.050692802667617796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,127,0.042499199509620667
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,255,0.08111519813537597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,255,0.050809597969055174
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,511,0.149782395362854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,511,0.08370720148086548
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,1023,0.1535920023918152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,1023,0.2863759994506836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,1,0.00886240005493164
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,1,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,3,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,3,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,7,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,7,0.009904000163078307
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,15,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,15,0.010531199723482132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,31,0.010516799986362457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,31,0.01043040007352829
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,63,0.009703999757766724
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,1023,0.08409439921379089
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,63,0.01045600026845932
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,127,0.01040000021457672
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,255,0.010401599854230881
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,255,0.008584000170230865
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,511,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,511,0.008507200330495835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,15,0.024934400618076325
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,1023,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,2047,0.010764800012111664
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,2047,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,4095,0.012582400441169738
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,4095,0.012604799866676331
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,8191,0.01459999978542328
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,8191,0.014124800264835358
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,16383,0.030144000053405763
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,16383,0.016648000478744505
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,32767,0.047440001368522645
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,32767,0.031119999289512635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,65535,0.08197919726371765
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,65535,0.04764960110187531
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,1,0.008454400300979614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,1,0.008601599931716919
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,3,0.008542399853467941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,3,0.008604799956083297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,7,0.0084927998483181
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,7,0.008635199815034866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,15,0.008513599634170532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,15,0.008646400272846222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,31,0.00857120007276535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,63,0.00862239971756935
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,63,0.008590400218963623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,127,0.008633600175380706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,127,0.008462399989366532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,127,0.010129600018262862
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,255,0.008504000306129456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,511,0.0086496002972126
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,511,0.008614400029182434
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,1023,0.010444799810647965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,31,0.008550400286912918
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,1023,0.00851840004324913
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,2047,0.010576000064611435
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,2047,0.010471999645233154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,4095,0.012665599584579468
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,4095,0.010699199885129929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,255,0.008857599645853042
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,8191,0.015028800070285796
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,16383,0.045998400449752806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,16383,0.02850080132484436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,32767,0.0796239972114563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,32767,0.046316799521446225
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,65535,0.14749280214309693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,65535,0.08159040212631226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,1,0.009468799829483033
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,8191,0.027142399549484254
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,1,0.008529599756002426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,3,0.010320000350475311
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,7,0.009089600294828415
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,3,0.008491200208663941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,15,0.010414399951696397
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,15,0.009948799759149552
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,31,0.010387200117111205
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,63,0.01014079973101616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,63,0.009467200189828873
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,127,0.009011200070381165
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,127,0.008575999736785888
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,255,0.009139200299978256
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,255,0.008486399799585343
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,511,0.010444799810647965
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,511,0.009108799695968627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,1023,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,1023,0.01053439974784851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,2047,0.012624000012874604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,2047,0.010580799728631973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,4095,0.027059200406074523
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,4095,0.013279999792575835
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,8191,0.04529280066490173
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,8191,0.027723199129104613
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,16383,0.07920160293579101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,16383,0.04536960124969482
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,32767,0.14782240390777587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,7,0.010460799932479859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,32767,0.08005599975585938
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,31,0.010335999727249145
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,65535,0.28467679023742676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,65535,0.14857759475708007
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,1,0.008587200194597244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,1,0.008548799902200699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,131071,0.556764793395996
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,131071,0.28614718914031984
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,7,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,7,0.010435199737548828
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,15,0.010401599854230881
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,15,0.01037919968366623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,31,0.010395199805498124
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,31,0.008640000224113464
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,63,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,63,0.008535999804735184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,1023,0.010583999752998351
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,127,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,255,0.010447999835014344
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,255,0.008609600365161896
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,511,0.010351999849081039
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,511,0.008535999804735184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,1023,0.010515200346708298
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,1023,0.0084927998483181
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,2047,0.010391999781131745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,2047,0.010412800312042236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,3,0.00857120007276535
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,4095,0.010664000362157821
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,4095,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,8191,0.012603199481964112
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,3,0.008561599999666214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,8191,0.012759999930858612
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,16383,0.01674720048904419
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,16383,0.016663999855518342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,32767,0.01685599982738495
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,32767,0.01671839952468872
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,65535,0.020771199464797975
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,65535,0.01733279973268509
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,131071,0.02070080041885376
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,1,0.008476799726486206
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,1,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,3,0.008580800145864487
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,3,0.010273600369691849
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,7,0.008529599756002426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,7,0.01048320010304451
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,15,0.008495999872684479
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,15,0.01048159971833229
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,31,0.008502399921417237
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,127,0.010449600219726563
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,31,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,63,0.010407999902963639
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,127,0.010398399829864503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,255,0.010972800105810166
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,255,0.008595199882984161
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,511,0.010334400087594986
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,511,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,1023,0.010315199941396713
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,131071,0.022819200158119203
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,1023,0.010425599664449692
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,2047,0.01053600013256073
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,2047,0.010608000308275222
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,4095,0.012601600587368011
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,63,0.008638399839401244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,4095,0.01072160005569458
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,8191,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,8191,0.014678399264812469
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,16383,0.0187376007437706
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,16383,0.01871519982814789
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,32767,0.018755200505256652
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,32767,0.018369600176811218
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,65535,0.020472000539302825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,65535,0.020790399610996248
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,131071,0.0375328004360199
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,131071,0.022793599963188173
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,1,0.008564800024032593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,1,0.008604799956083297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,3,0.00852160006761551
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,7,0.008516799658536911
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,7,0.008635199815034866
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,15,0.00862559974193573
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,15,0.00870240032672882
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,31,0.008585599809885025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,31,0.008539199829101562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,63,0.00860000029206276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,63,0.008657599985599517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,127,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,127,0.01045759990811348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,127,0.008590400218963623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,255,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,255,0.008591999858617782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,511,0.009440000355243682
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,511,0.008488000184297562
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,1023,0.010492800176143647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,2047,0.010569600015878677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,2047,0.010487999767065048
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,4095,0.012585599720478059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,4095,0.012476799637079239
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,8191,0.026926401257514953
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,8191,0.014571200311183929
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,16383,0.04510720074176788
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,16383,0.028676798939704894
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,32767,0.07943999767303467
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,32767,0.04584639966487884
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,65535,0.1471328020095825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,65535,0.08040159940719604
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,131071,0.1492624044418335
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,131071,0.28456480503082277
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,1,0.010503999888896942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,1,0.010411199927330018
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,3,0.010407999902963639
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,3,0.010238400101661682
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,7,0.009622400254011154
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,15,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,15,0.010518400371074677
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,31,0.0105103999376297
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,31,0.010451199859380722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,63,0.010467199981212616
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,1023,0.009249600023031235
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,63,0.008560000360012055
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,127,0.010806400328874588
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,127,0.009027200192213059
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,3,0.008470399677753449
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,511,0.010369600355625152
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,511,0.010265599936246872
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,1023,0.010398399829864503
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,1023,0.010396800190210342
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,2047,0.010416000336408614
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,2047,0.010543999820947647
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,4095,0.012464000284671784
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,4095,0.011137600243091583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,8191,0.014585599303245544
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,8191,0.01311360001564026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,16383,0.016438399255275727
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,16383,0.014716799557209014
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,7,0.010441599786281586
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,32767,0.018716800212860107
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,32767,0.017584000527858735
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,65535,0.018801599740982056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,131071,0.051979202032089236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,131071,0.034641599655151366
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,1,0.008529599756002426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,1,0.008590400218963623
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,3,0.00856959968805313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,3,0.008513599634170532
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,255,0.008489599823951722
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,7,0.008556800335645676
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,7,0.008478400111198426
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,15,0.009142400324344635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,15,0.008510400354862214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,31,0.008500800281763077
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,63,0.00854559987783432
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,63,0.008419200032949447
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,127,0.008428800106048583
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,127,0.008449599891901017
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,255,0.008444800227880477
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,255,0.008401600271463394
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,511,0.010571199655532836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,65535,0.03422400057315826
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,511,0.008828800171613693
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,1023,0.012476799637079239
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,1023,0.010427200049161912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,2047,0.027063998579978942
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,2047,0.012646399438381195
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,4095,0.04514079988002777
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,4095,0.027504000067710876
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,8191,0.07905759811401367
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,255,0.01071999967098236
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,8191,0.045363199710845944
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,16383,0.08006240129470825
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,31,0.00889120027422905
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,16383,0.147051203250885
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,32767,0.28385119438171386
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,65535,0.556828784942627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,65535,0.28761920928955076
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,1,0.010675200074911118
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,1,0.010462400317192078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,3,0.010635200142860412
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,7,0.010649599879980088
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,7,0.010443200170993806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,15,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,15,0.010480000078678131
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,31,0.010572800040245056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,31,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,63,0.010731200128793717
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,63,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,127,0.01067039966583252
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,32767,0.1482591986656189
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,127,0.010478399693965912
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,255,0.01053759977221489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,255,0.010622400045394897
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,511,0.012825599312782288
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,511,0.012507200241088867
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,1023,0.027531200647354127
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,2047,0.04548160135746002
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,2047,0.027236801385879517
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,4095,0.0800383985042572
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,4095,0.04562880098819733
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,8191,0.14833439588546754
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,8191,0.08101599812507629
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,16383,0.28618080615997316
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,1,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,1,0.012457600235939026
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,3,0.01266240030527115
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,3,0.01255040019750595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,7,0.012691199779510498
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,7,0.01249760016798973
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,15,0.012601600587368011
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,15,0.012555199861526489
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,31,0.012851199507713318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,31,0.012492799758911132
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,63,0.01263200044631958
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,63,0.012468799948692322
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,3,0.010542400181293488
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,127,0.012614400684833526
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,255,0.0126351997256279
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,255,0.01250240057706833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,511,0.028519999980926514
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,511,0.014684799313545226
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,16383,0.15079360008239745
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,1023,0.029287999868392943
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,2047,0.08109120130538941
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,2047,0.04779999852180481
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,4095,0.15218720436096192
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,4095,0.08217120170593262
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,1023,0.01467359960079193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,8191,0.29304640293121337
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,8191,0.1512928009033203
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,127,0.012608000636100769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,1,0.01019359976053238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,1,0.010691200196743012
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,3,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,3,0.009412799775600434
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,7,0.010590399801731109
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,7,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,15,0.008564800024032593
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,31,0.010491199791431427
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,63,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,1023,0.04607360064983368
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,63,0.008459199965000153
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,127,0.010475199669599533
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,127,0.008555199950933456
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,255,0.010409600287675857
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,511,0.0104592002928257
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,511,0.010454399883747101
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,1023,0.010507199913263321
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,1023,0.010438399761915207
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,2047,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,2047,0.010526400059461594
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,4095,0.012544000148773193
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,4095,0.0125231996178627
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,8191,0.014611199498176575
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,8191,0.012705600261688233
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,16383,0.01680160015821457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,16383,0.014796799421310425
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,15,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,32767,0.03250080049037933
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,32767,0.01669279932975769
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,65535,0.04954720139503479
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,65535,0.03307999968528748
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,255,0.008612799644470214
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,131071,0.0836143970489502
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,131071,0.04947839975357056
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,1,0.01764799952507019
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,1,0.016780799627304076
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,3,0.016652800142765045
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,7,0.016832000017166136
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,7,0.016599999368190767
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,15,0.016774399578571318
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,15,0.016684800386428833
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,31,0.016864000260829924
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,31,0.01671359986066818
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,63,0.016896000504493712
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,63,0.016655999422073364
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,127,0.016857600212097167
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,31,0.009700799733400345
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,127,0.016777600347995757
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,255,0.027057600021362305
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,255,0.016697600483894348
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,511,0.045459198951721194
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,1023,0.07939680218696595
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,3,0.016939200460910797
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,1023,0.04687199890613556
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,2047,0.08138560056686402
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,4095,0.28443520069122313
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,4095,0.149782395362854
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,1,0.027075201272964478
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,1,0.02489279955625534
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,3,0.026919999718666078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,3,0.024830399453639983
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,7,0.026947200298309326
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,511,0.028942400217056276
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,7,0.024875199794769286
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,15,0.02492000013589859
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,31,0.02694239914417267
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,15,0.027451199293136597
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,31,0.024855999648571013
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,63,0.027014398574829103
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,63,0.025366398692131042
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,127,0.024966399371623992
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,255,0.031139200925827025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,255,0.04593920111656189
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,2047,0.14822720289230346
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,511,0.08050400018692017
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,511,0.04868800044059753
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,1023,0.1482143998146057
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,1023,0.08335840106010436
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,1,0.010468800365924836
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,2047,0.2853935956954956
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,2047,0.15166879892349244
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,1,0.008686400204896926
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,7,0.010505600273609162
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,3,0.010446400195360184
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,7,0.008617600053548813
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,15,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,15,0.008548799902200699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,31,0.010363200306892395
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,31,0.008540800213813782
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,63,0.010344000160694122
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,63,0.008497600257396699
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,127,0.010463999956846238
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,127,0.008816000074148178
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,255,0.01053439974784851
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,255,0.008895999938249587
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,511,0.010496000200510025
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,511,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,1023,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,1023,0.010513599961996078
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,2047,0.010847999900579452
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,2047,0.010567999631166457
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,4095,0.012673600018024445
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,4095,0.012556800246238708
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,8191,0.014824000000953675
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,127,0.03025600016117096
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,16383,0.030819201469421388
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,16383,0.014662399888038635
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,32767,0.04749920070171356
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,32767,0.02961440086364746
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,65535,0.08219199776649475
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,65535,0.04732959866523743
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,131071,0.1511247992515564
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,131071,0.08280320167541504
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,3,0.010360000282526016
SGLang,0.5.9,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,8191,0.012596799433231354
