framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,1,0.04561919867992401
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,3,0.04651040136814118
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,1,0.03922080099582672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,3,0.03900319933891296
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,7,0.039027199149131775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,7,0.045592001080513
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,15,0.04666880071163178
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,15,0.038596799969673155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,31,0.04738079905509949
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,31,0.039043200016021726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,63,0.049660798907279965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,127,0.055550402402877806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,127,0.04005120098590851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,255,0.04743359982967377
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,255,0.0841871976852417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,511,0.08220000267028808
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,511,0.15247999429702758
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,1023,0.153547203540802
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,1023,0.28924639225006105
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,2047,0.29990079402923586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,float16,2047,0.5672063827514648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,1,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,3,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,1,0.008556800335645676
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,3,0.009345600008964538
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,7,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,7,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,15,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,15,0.008475200086832047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,31,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,31,0.008433599770069123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,63,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,63,0.008470399677753449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,127,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,255,0.010540799796581268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,255,0.010174400359392165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,511,0.010531199723482132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,511,0.008720000088214875
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,1023,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,1023,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,fp8,2047,0.010494399815797806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,64,128,1,float16,float16,2047,0.014246399700641631
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,1,0.0105103999376297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,1,0.008428800106048583
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,3,0.010489600151777268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,3,0.008423999696969987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,7,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,7,0.008441600203514098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,15,0.010857599973678588
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,15,0.008372800052165985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,31,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,31,0.008473599702119828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,63,0.010516799986362457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,63,0.00843999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,127,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,255,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,255,0.008408000320196151
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,127,0.008454400300979614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,511,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,511,0.009083200246095657
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,1023,0.012548799812793731
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,1023,0.010468800365924836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,float16,2047,0.030990400910377504
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,64,128,1,float16,fp8,2047,0.01255040019750595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,1,0.02887359857559204
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,1,0.022988800704479218
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,3,0.02881920039653778
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,7,0.028908801078796387
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,3,0.02272319942712784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,7,0.022758400440216063
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,15,0.02877120077610016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,15,0.022784000635147093
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,31,0.02279040068387985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,31,0.028814399242401124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,63,0.027505600452423097
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,63,0.022745600342750548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,127,0.033827200531959534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,127,0.02276639938354492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,255,0.048902401328086854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,255,0.027580800652503967
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,511,0.08355039954185486
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,511,0.04575360119342804
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,1023,0.0823248028755188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,1023,0.15185439586639404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,64,128,1,float16,fp8,63,0.03872799873352051
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,fp8,2047,0.15478559732437133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,64,128,1,float16,float16,2047,0.2910239934921265
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,1,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,3,0.012470400333404541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,3,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,7,0.012303999811410903
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,7,0.010361599922180175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,15,0.012438400089740754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,15,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,31,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,63,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,31,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,63,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,127,0.012491200119256973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,127,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,255,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,255,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,1,0.012444800138473511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,511,0.014550399780273438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,1023,0.02959200143814087
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,511,0.01056160032749176
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,1023,0.014561599493026734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,fp8,2047,0.028679999709129333
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,64,128,1,float16,float16,2047,0.04873600006103516
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,1,0.06992319822311402
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,3,0.08223680257797242
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,3,0.06977599859237671
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,7,0.08231359720230103
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,7,0.06981599926948548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,15,0.06985120177268982
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,15,0.08625919818878174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,31,0.08630560040473938
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,63,0.08708639740943909
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,31,0.06984959840774536
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,63,0.06989439725875854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,127,0.09265279769897461
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,127,0.07035679817199707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,255,0.08448479771614074
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,255,0.15357600450515746
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,511,0.2908143997192383
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,511,0.15324959754943848
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,fp8,1023,0.29654879570007325
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,1023,0.5639440059661865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,1,0.15517120361328124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,64,128,1,float16,float16,1,0.08247519731521606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,1,0.13125120401382445
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,3,0.15616480112075806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,7,0.16016319990158082
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,15,0.16018879413604736
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,7,0.12928320169448854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,15,0.1312880039215088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,31,0.15979520082473755
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,31,0.13131999969482422
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,63,0.16059199571609498
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,63,0.1312608003616333
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,127,0.17279360294342042
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,127,0.1323456048965454
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,float16,255,0.2933552026748657
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,255,0.1589184045791626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,1,0.2522752046585083
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,1,0.3016144037246704
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,3,0.30660319328308105
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,3,0.2522783994674683
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,7,0.30369279384613035
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,7,0.2531008005142212
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,15,0.3073456048965454
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,15,0.25193119049072266
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,31,0.25396161079406737
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,31,0.3060143947601318
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,63,0.3058367967605591
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,63,0.2534303903579712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,1,0.014556799829006196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,float16,127,0.3304768085479736
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,1,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,64,128,1,float16,fp8,127,0.25247199535369874
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,3,0.012350399792194367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,7,0.014470399916172027
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,3,0.014603200554847717
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,7,0.013953599333763122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,15,0.014580799639225006
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,31,0.014472000300884247
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,15,0.012643200159072877
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,31,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,63,0.014526399970054626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,63,0.012625600397586822
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,127,0.014472000300884247
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,127,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,255,0.014579200744628906
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,511,0.03030399978160858
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,255,0.012455999851226807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,1023,0.04971520006656647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,511,0.014689600467681885
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,1023,0.0289247989654541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,fp8,2047,0.04817599952220917
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,64,128,1,float16,float16,2047,0.08601120114326477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,1,0.4978640079498291
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,1,0.5963312149047851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,3,0.6010191917419434
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,3,0.49817919731140137
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,7,0.4984303951263428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,7,0.5943568229675293
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,15,0.49818878173828124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,15,0.5999216079711914
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,31,0.59617919921875
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,31,0.497814416885376
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,float16,63,0.601582384109497
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,64,128,1,float16,fp8,63,0.49321599006652833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,1,1.1951968193054199
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,1,0.9873344421386718
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,3,1.1842432022094727
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,3,0.9888768196105957
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,7,0.9796496391296386
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,7,1.1913135528564454
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,64,128,1,float16,fp8,3,0.12940640449523927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,15,1.1931983947753906
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,15,0.9784015655517578
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,1,0.01863359957933426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,float16,31,1.1926544189453125
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,1,0.01465120017528534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,3,0.014670400321483612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,3,0.018593600392341612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,7,0.018713599443435668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,7,0.014519999921321868
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,15,0.018681600689888
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,15,0.014569599926471711
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,31,0.01873600035905838
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,31,0.014539200067520141
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,63,0.01871200054883957
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,63,0.014564800262451171
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,127,0.018743999302387238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,64,128,1,float16,fp8,31,0.9852255821228028
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,127,0.014641599357128143
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,255,0.031031998991966247
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,255,0.014585599303245544
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,511,0.04932959973812103
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,511,0.028883200883865357
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,1023,0.047019198536872864
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,1023,0.08195199966430664
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,fp8,2047,0.08459680080413819
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,64,128,1,float16,float16,2047,0.1521504044532776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,1,0.037099200487136844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,1,0.03097119927406311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,3,0.031118398904800414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,7,0.03103039860725403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,15,0.03711679875850678
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,7,0.03707039952278137
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,15,0.03096800148487091
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,31,0.037083199620246886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,31,0.030956798791885377
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,63,0.031044799089431762
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,63,0.03931039869785309
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,127,0.04351359903812409
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,127,0.03152799904346466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,255,0.06569439768791199
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,255,0.03748959898948669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,511,0.06416959762573242
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,511,0.11666239500045776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,1023,0.11762399673461914
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,1023,0.22005600929260255
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,1,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,3,0.037049600481987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,3,0.010582400113344192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,fp8,2047,0.22543840408325194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,7,0.010500799864530563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,1,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,15,0.010580799728631973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,15,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,31,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,31,0.010499200224876404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,7,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,63,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,63,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,127,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,127,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,255,0.010516799986362457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,255,0.010492800176143647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,511,0.011214400082826615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,511,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,1023,0.012520000338554382
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,1023,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,float16,2047,0.014577600359916686
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,2047,0.012520000338554382
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,1,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,1,0.008433599770069123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,3,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,3,0.008436799794435502
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,7,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,48,48,128,1,float16,fp8,3,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,7,0.008395200222730636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,15,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,31,0.01016639992594719
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,15,0.008388800173997879
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,31,0.008401600271463394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,63,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,63,0.008526399731636047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,127,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,127,0.008430399745702744
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,255,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,48,48,128,1,float16,float16,2047,0.42762079238891604
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,255,0.00843999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,511,0.01058880016207695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,511,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,1023,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,2047,0.020641599595546723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,fp8,2047,0.01268800050020218
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,48,48,128,1,float16,float16,1023,0.012622399628162384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,1,0.02279040068387985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,1,0.020336000621318816
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,3,0.022830399870872497
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,7,0.02269120067358017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,7,0.018760000169277192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,15,0.018615999817848207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,15,0.022873599827289582
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,31,0.022729599475860597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,31,0.01865919977426529
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,63,0.022835199534893037
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,63,0.018636800348758698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,127,0.026785600185394286
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,127,0.018606400489807128
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,255,0.039268800616264345
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,255,0.02062080055475235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,511,0.06583840250968934
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,511,0.037390398979187014
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,1023,0.06431679725646973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,1023,0.11856640577316284
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,2047,0.11821600198745727
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,fp8,3,0.02064319998025894
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,48,48,128,1,float16,float16,2047,0.22525761127471924
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,1,0.010585600137710571
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,1,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,3,0.0097680002450943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,3,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,7,0.010230399668216705
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,15,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,7,0.01063840016722679
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,31,0.010521599650382995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,15,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,31,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,63,0.010332799702882766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,127,0.010599999874830245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,63,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,127,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,255,0.010603199899196624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,255,0.008372800052165985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,511,0.014475199580192565
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,511,0.010550399869680404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,1023,0.021087999641895293
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,1023,0.014572800695896148
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,fp8,2047,0.020177599787712098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,48,48,128,1,float16,float16,2047,0.0430400013923645
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,1,0.06416000127792358
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,1,0.05356799960136414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,3,0.06393120288848878
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,3,0.053521597385406496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,7,0.06371200084686279
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,15,0.06375679969787598
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,7,0.05355520248413086
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,15,0.05343679785728454
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,31,0.06785600185394287
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,31,0.05360640287399292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,63,0.06798560023307801
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,63,0.053598397970199586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,127,0.05572479963302612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,127,0.07323520183563233
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,255,0.1175663948059082
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,255,0.06593599915504456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,511,0.2207200050354004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,511,0.1172719955444336
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,1,0.1190224051475525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,float16,1023,0.4252960205078125
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,1,0.09858880043029786
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,48,48,128,1,float16,fp8,1023,0.22539360523223878
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,3,0.09866560101509095
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,3,0.11900479793548584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,7,0.09871519804000854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,7,0.12111200094223022
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,15,0.12162239551544189
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,15,0.09854879975318909
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,31,0.10064799785614013
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,31,0.12332960367202758
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,63,0.12336479425430298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,63,0.10064799785614013
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,127,0.10271680355072021
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,127,0.1329967975616455
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,float16,255,0.22215359210968016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,48,48,128,1,float16,fp8,255,0.12209279537200927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,1,0.22903358936309814
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,1,0.19138720035552978
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,3,0.22968640327453613
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,3,0.19088000059127808
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,7,0.23183839321136473
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,7,0.19057919979095458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,15,0.23181440830230712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,15,0.19194560050964354
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,31,0.2313152074813843
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,31,0.19138879776000978
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,63,0.23268160820007325
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,63,0.1915071964263916
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,1,0.010577599704265594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,float16,127,0.2506047964096069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,1,0.012755200266838074
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,3,0.012624000012874604
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,48,48,128,1,float16,fp8,127,0.19361599683761596
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,7,0.012715199589729309
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,3,0.010492800176143647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,7,0.010492800176143647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,31,0.012563200294971466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,15,0.012529599666595458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,15,0.010704000294208527
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,31,0.01056319996714592
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,63,0.012574400007724761
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,63,0.01050880029797554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,127,0.010553599894046783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,127,0.012535999715328216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,255,0.01249760016798973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,255,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,511,0.01889760047197342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,511,0.012563200294971466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,1023,0.018615999817848207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,1023,0.039113599061965945
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,float16,2047,0.06673759818077088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,48,48,128,1,float16,fp8,2047,0.03770720064640045
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,1,0.45131359100341795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,1,0.3748575925827026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,3,0.4489888191223145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,3,0.3752608060836792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,7,0.450929594039917
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,7,0.37304160594940183
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,15,0.45139517784118655
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,15,0.37523839473724363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,31,0.4491936206817627
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,31,0.3749808073043823
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,float16,63,0.45336642265319826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,48,48,128,1,float16,fp8,63,0.3733583927154541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,1,0.8949775695800781
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,1,0.742241621017456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,3,0.8887935638427734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,3,0.7424831867218018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,7,0.8932880401611328
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,7,0.7372447967529296
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,15,0.8944432258605957
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,15,0.7403327941894531
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,float16,31,0.8895152091979981
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,1,0.016627199947834015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,3,0.01645279973745346
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,1,0.014433600008487701
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,3,0.014428800344467163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,7,0.01648160070180893
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,7,0.014534400403499603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,15,0.016704000532627106
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,15,0.014444799721240997
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,31,0.016689600050449373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,31,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,63,0.016599999368190767
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,48,48,128,1,float16,fp8,31,0.7389039993286133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,63,0.014431999623775482
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,255,0.020510399341583253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,127,0.01454399973154068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,127,0.01672479957342148
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,255,0.014123199880123139
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,511,0.04051199853420258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,511,0.018772800266742707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,1023,0.06773120164871216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,1023,0.037363201379776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,fp8,2047,0.06567839980125427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,48,48,128,1,float16,float16,2047,0.12326879501342773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,1,0.031436800956726074
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,7,0.031046399474143983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,3,0.03212960064411163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,15,0.03246560096740723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,1,0.02689119875431061
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,31,0.03269599974155426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,31,0.026815998554229736
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,63,0.033769598603248595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,63,0.02688319981098175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,127,0.037084800004959104
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,127,0.026824000477790832
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,255,0.05570560097694397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,255,0.03429119884967804
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,15,0.026939201354980468
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,511,0.09865440130233764
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,511,0.055508798360824584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,1023,0.10065280199050904
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,1023,0.18486239910125732
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,2047,0.1906208038330078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,float16,2047,0.3560879945755005
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,1,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,3,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,3,0.026907199621200563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,7,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,40,40,128,1,float16,fp8,7,0.026766398549079896
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,15,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,3,0.008454400300979614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,31,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,1,0.008433599770069123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,31,0.00841279998421669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,63,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,63,0.008416000008583068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,127,0.01053600013256073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,127,0.008376000076532364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,255,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,255,0.008399999886751174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,511,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,511,0.008403199911117553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,1023,0.010664000362157821
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,1023,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,float16,2047,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,2047,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,1,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,1,0.008380799740552902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,3,0.010228800028562546
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,3,0.008404800295829773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,7,0.010121600329875946
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,7,0.008320000022649765
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,15,0.01003360003232956
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,15,0.008379200100898742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,31,0.009822399914264679
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,31,0.008419200032949447
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,63,0.009753599762916565
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,63,0.00835679993033409
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,127,0.008516799658536911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,127,0.008364800363779068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,255,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,255,0.008398400247097015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,511,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,511,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,15,0.008540800213813782
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,40,40,128,1,float16,fp8,7,0.008683200180530547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,1023,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,1023,0.010556799918413162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,float16,2047,0.016998399794101716
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,40,40,128,1,float16,fp8,2047,0.012561599910259246
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,1,0.020824000239372253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,1,0.0186256006360054
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,3,0.020670400559902193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,3,0.018568000197410582
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,7,0.020740799605846405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,7,0.016663999855518342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,15,0.02083359956741333
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,15,0.018723200261592864
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,31,0.020683200657367708
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,31,0.018694399297237395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,63,0.020919999480247496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,63,0.01660960018634796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,127,0.022724799811840057
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,127,0.018665599822998046
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,255,0.03325600028038025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,255,0.01857440024614334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,511,0.032924801111221313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,511,0.056195199489593506
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,1023,0.09881439805030823
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,1023,0.05527359843254089
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,1,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,float16,2047,0.18587199449539185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,40,40,128,1,float16,fp8,2047,0.10040639638900757
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,1,0.008980800211429597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,3,0.010577599704265594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,3,0.008486399799585343
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,7,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,7,0.008588799834251403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,15,0.010604800283908844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,15,0.008403199911117553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,31,0.010556799918413162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,31,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,63,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,127,0.01056319996714592
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,63,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,127,0.00883520022034645
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,511,0.012462399899959564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,255,0.010531199723482132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,255,0.008502399921417237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,511,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,1023,0.016766400635242464
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,1023,0.012780800461769104
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,float16,2047,0.041140800714492796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,40,40,128,1,float16,fp8,2047,0.018564799427986146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,1,0.04530400037765503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,1,0.05554720163345337
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,3,0.054094398021698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,3,0.04528000056743622
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,7,0.055111998319625856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,7,0.04536640048027039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,15,0.054604798555374146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,15,0.04694559872150421
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,31,0.057543998956680296
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,31,0.045263999700546266
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,63,0.05971840023994446
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,63,0.046481600403785704
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,127,0.06200640201568604
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,127,0.04830879867076874
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,255,0.09969919919967651
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,255,0.05751039981842041
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,511,0.18549120426177979
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,511,0.09955999851226807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,1,0.10114560127258301
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,float16,1023,0.35554559230804444
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,1,0.08424479961395263
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,40,40,128,1,float16,fp8,1023,0.18996479511260986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,3,0.08420000076293946
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,3,0.10066239833831787
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,7,0.10115679502487182
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,7,0.08419839739799499
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,15,0.10381120443344116
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,15,0.08431839942932129
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,31,0.10307519435882569
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,31,0.0843616008758545
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,63,0.10528800487518311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,63,0.0848688006401062
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,127,0.11084799766540528
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,127,0.08689600229263306
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,float16,255,0.1870319962501526
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,40,40,128,1,float16,fp8,255,0.10586559772491455
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,1,0.19145599603652955
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,1,0.16208479404449463
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,3,0.19130079746246337
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,3,0.16002559661865234
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,7,0.19490400552749634
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,7,0.16005280017852783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,15,0.16196800470352174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,15,0.19490879774093628
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,31,0.16208640336990357
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,31,0.1945296049118042
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,63,0.19521119594573974
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,63,0.16216640472412108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,fp8,127,0.16326719522476196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,1,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,40,40,128,1,float16,float16,127,0.20962400436401368
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,1,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,3,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,3,0.01055999994277954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,7,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,15,0.012510399520397186
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,15,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,7,0.010583999752998351
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,31,0.01255359947681427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,31,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,63,0.012427199631929398
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,63,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,127,0.012398400157690049
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,127,0.010553599894046783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,255,0.012404800206422806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,511,0.016128000617027283
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,255,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,511,0.012579199671745301
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,1023,0.03717760145664215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,1023,0.0166143998503685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,float16,2047,0.06164000034332275
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,40,40,128,1,float16,fp8,2047,0.035627201199531555
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,1,0.37699840068817136
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,1,0.3116735935211182
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,3,0.37757918834686277
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,3,0.3136784076690674
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,7,0.375217604637146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,7,0.31409759521484376
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,15,0.37758560180664064
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,15,0.3118223905563354
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,31,0.3777440071105957
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,31,0.313753604888916
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,float16,63,0.37720479965209963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,40,40,128,1,float16,fp8,63,0.3138479948043823
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,1,0.7471504211425781
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,1,0.616374397277832
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,3,0.7469727993011475
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,3,0.6206624031066894
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,7,0.7419263839721679
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,7,0.6201072216033936
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,15,0.7475887775421143
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,15,0.615231990814209
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,1,0.014500799775123595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,1,0.012593600153923034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,3,0.014504000544548035
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,float16,31,0.7469679832458496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,3,0.012600000202655792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,7,0.014587199687957764
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,40,40,128,1,float16,fp8,31,0.6199791908264161
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,15,0.014619199931621552
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,7,0.012601600587368011
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,15,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,31,0.012627199292182922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,63,0.012462399899959564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,31,0.014752000570297241
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,63,0.014636799693107605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,127,0.014571200311183929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,127,0.012548799812793731
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,255,0.01656640022993088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,255,0.01260959953069687
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,511,0.03503200113773346
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,511,0.016631999611854555
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,1023,0.033430400490760806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,1023,0.05732799768447876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,1,0.026903998851776124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,fp8,2047,0.05634719729423523
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,40,40,128,1,float16,float16,2047,0.10280799865722656
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,3,0.026870399713516235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,7,0.026867198944091796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,15,0.02694559991359711
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,1,0.022759999334812164
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,31,0.026811200380325317
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,31,0.02276960015296936
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,63,0.027030399441719054
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,63,0.022812800109386445
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,127,0.031225600838661195
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,127,0.022707200050354003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,255,0.04731520116329193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,255,0.02879680097103119
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,511,0.08079040050506592
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,511,0.04660640060901642
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,1023,0.14979039430618285
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,1023,0.08230400085449219
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,2047,0.2884511947631836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,2047,0.155622398853302
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,3,0.022731199860572815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,float16,4095,0.5658880233764648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,1,0.009777600318193436
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,4095,0.3056256055831909
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,7,0.02298399955034256
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,3,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,3,0.00846560001373291
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,1,0.008406399935483932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,7,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,7,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,15,0.010310400277376175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,15,0.00840959995985031
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,31,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,31,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,63,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,63,0.00849440023303032
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,127,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,127,0.008529599756002426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,255,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,255,0.008523199707269669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,511,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,511,0.00854720026254654
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,1023,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,1023,0.008657599985599517
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,2047,0.010513599961996078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,2047,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,float16,4095,0.012569600343704223
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,1,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,32,128,1,float16,fp8,4095,0.010526400059461594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,1,0.008473599702119828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,3,0.010494399815797806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,3,0.008472000062465668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,7,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,7,0.008423999696969987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,15,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,31,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,15,0.008417599648237229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,31,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,63,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,127,0.01035040020942688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,63,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,127,0.008662399649620057
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,255,0.01058880016207695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,255,0.00867839977145195
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,511,0.010540799796581268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,511,0.00881119966506958
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,2047,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,1023,0.010755199939012527
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,1023,0.010543999820947647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,2047,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,float16,4095,0.02887200117111206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,1,0.018598400056362152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,32,128,1,float16,fp8,4095,0.01465280055999756
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,1,0.014724799990653991
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,3,0.017425599694252013
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,7,0.018580800294876097
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,3,0.014776000380516052
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,7,0.014584000408649444
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,15,0.01857919991016388
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,31,0.0182096004486084
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,15,0.014737600088119506
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,31,0.016502399742603303
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,63,0.0182096004486084
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,63,0.01525759994983673
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,127,0.01858240067958832
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,127,0.014571200311183929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,255,0.016198399662971496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,255,0.029241600632667543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,32,128,1,float16,fp8,15,0.022908799350261688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,511,0.046033599972724916
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,511,0.028673601150512696
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,1023,0.08141599893569947
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,1023,0.04638879895210266
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,2047,0.15010559558868408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,2047,0.0843936026096344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,1,0.00846719965338707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,1,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,3,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,3,0.008497600257396699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,7,0.010326399654150008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,7,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,float16,4095,0.2894992113113403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,15,0.010260800272226334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,32,128,1,float16,fp8,4095,0.15964159965515137
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,15,0.008483199775218964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,31,0.008444800227880477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,31,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,63,0.008582399785518646
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,63,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,127,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,255,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,127,0.008558399975299835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,255,0.008404800295829773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,511,0.010550399869680404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,511,0.008803199976682663
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,1023,0.012518399953842163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,1023,0.010515200346708298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,2047,0.028958401083946227
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,2047,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,fp8,4095,0.027184000611305235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,32,128,1,float16,float16,4095,0.04538719952106476
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,1,0.045403200387954715
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,1,0.03742879927158356
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,3,0.04522239863872528
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,3,0.038996800780296326
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,7,0.045407998561859134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,7,0.03852800130844116
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,15,0.04521760046482086
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,15,0.03917439877986908
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,31,0.0472896009683609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,31,0.03911679983139038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,63,0.049409601092338565
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,63,0.03909600079059601
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,127,0.051342397928237915
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,127,0.04011679887771606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,255,0.04738079905509949
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,255,0.08179200291633607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,511,0.15038559436798096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,511,0.0820576012134552
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,1023,0.2870879888534546
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,1023,0.15418239831924438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,1,0.08220319747924805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,1,0.06979519724845887
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,float16,2047,0.5611743927001953
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,3,0.08222240209579468
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,32,128,1,float16,fp8,2047,0.3005759954452515
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,7,0.0823248028755188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,3,0.06982399821281433
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,15,0.08433279991149903
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,7,0.06994079947471618
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,15,0.06984639763832093
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,31,0.08523839712142944
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,31,0.06978240013122558
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,63,0.08611519932746887
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,63,0.06999359726905822
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,127,0.09098079800605774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,127,0.07180320024490357
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,255,0.08606399893760681
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,255,0.15198080539703368
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,1,0.1549504041671753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,float16,511,0.2880880117416382
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,32,128,1,float16,fp8,511,0.1527184009552002
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,1,0.129312002658844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,3,0.15585440397262573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,3,0.12941759824752808
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,7,0.1568127989768982
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,7,0.13133759498596193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,15,0.1580415964126587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,15,0.13133599758148193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,31,0.15846560001373292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,31,0.1313264012336731
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,63,0.15840959548950195
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,63,0.13148000240325927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,127,0.168614399433136
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,127,0.13294240236282348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,1,0.010527999699115753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,1,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,float16,255,0.2911184072494507
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,3,0.01061440035700798
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,32,128,1,float16,fp8,255,0.16438560485839843
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,7,0.01053759977221489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,3,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,7,0.01003199964761734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,15,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,15,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,31,0.010492800176143647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,31,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,63,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,63,0.01029599979519844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,127,0.01173119992017746
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,127,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,255,0.01053439974784851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,255,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,511,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,511,0.010856000334024429
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,1023,0.028947201371192933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,1023,0.014324800670146942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,2047,0.04818240106105805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,2047,0.028433600068092348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,float16,4095,0.08262400031089782
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,32,128,1,float16,fp8,4095,0.04737119972705841
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,1,0.3016607999801636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,1,0.2524415969848633
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,3,0.3041248083114624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,3,0.2524384021759033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,7,0.30431039333343507
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,7,0.25412640571594236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,15,0.30352799892425536
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,15,0.25208001136779784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,31,0.3054415941238403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,31,0.2531696081161499
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,63,0.3055727958679199
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,63,0.2517407894134521
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,float16,127,0.32607839107513426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,32,128,1,float16,fp8,127,0.2563040018081665
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,1,0.4941455841064453
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,1,0.5995520114898681
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,3,0.4982016086578369
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,3,0.6006735801696778
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,7,0.49809918403625486
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,7,0.5960112094879151
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,15,0.5999343872070313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,15,0.4984111785888672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,31,0.49382719993591306
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,31,0.5997456073760986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,1,0.014019200205802917
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,1,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,float16,63,0.6024176120758057
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,3,0.013299199938774108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,3,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,7,0.013033600151538849
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,7,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,15,0.013073599338531494
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,32,128,1,float16,fp8,63,0.49390559196472167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,15,0.012409599870443344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,31,0.013748799264431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,63,0.013969600200653076
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,31,0.012452799826860428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,63,0.012435200065374375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,127,0.013732799887657165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,127,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,255,0.012851199507713318
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,255,0.011761599779129028
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,511,0.014721600711345673
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,511,0.029068800806999206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,1023,0.0472784012556076
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,1023,0.02917119860649109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,2047,0.08373919725418091
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,2047,0.04731679856777191
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,1,0.022694399952888487
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,float16,4095,0.15795040130615234
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,32,128,1,float16,fp8,4095,0.08420479893684388
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,3,0.022745600342750548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,7,0.022767999768257143
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,1,0.018742400407791137
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,15,0.02276480048894882
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,15,0.01863359957933426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,31,0.02264000028371811
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,31,0.019412800669670105
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,63,0.022720000147819518
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,63,0.019332799315452575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,127,0.024831999838352204
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,127,0.01881439983844757
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,255,0.037694400548934935
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,255,0.020777599513530733
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,511,0.0638704001903534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,511,0.03718239963054657
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,1023,0.11632959842681885
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,1023,0.06389920115470886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,2047,0.22107839584350586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,float16,4095,0.4322175979614258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,3,0.018719999492168425
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,1,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,4095,0.22662880420684814
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,3,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,1,0.010577599704265594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,3,0.010521599650382995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,7,0.010507199913263321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,7,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,15,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,15,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,31,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,31,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,7,0.020612800121307374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,63,0.01053439974784851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,63,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,127,0.010583999752998351
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,127,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,255,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,511,0.010606399923563003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,511,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,255,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,1023,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,1023,0.010673599690198899
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,2047,0.012670400738716125
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,2047,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,float16,4095,0.014451199769973755
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,24,24,128,1,float16,fp8,4095,0.012508800625801087
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,1,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,1,0.010320000350475311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,3,0.01055999994277954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,3,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,7,0.01055999994277954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,7,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,15,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,15,0.010344000160694122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,31,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,31,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,63,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,127,0.010590399801731109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,63,0.010489600151777268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,127,0.01035040020942688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,255,0.010489600151777268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,255,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,1023,0.012392000108957291
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,511,0.010945600271224976
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,511,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,2047,0.01239359974861145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,1023,0.011009600013494492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,2047,0.012622399628162384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,float16,4095,0.017019200325012206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,24,24,128,1,float16,fp8,4095,0.01451359987258911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,1,0.016491200029850005
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,1,0.014499199390411378
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,3,0.01650719940662384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,3,0.014531199634075165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,7,0.01658719927072525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,7,0.014510400593280792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,15,0.016153599321842193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,15,0.014496000111103058
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,31,0.016190400719642638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,31,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,63,0.01586720049381256
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,63,0.014446400105953217
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,127,0.016344000399112702
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,127,0.014427199959754944
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,255,0.018777599930763243
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,255,0.014467200636863709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,511,0.03906719982624054
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,511,0.01928640007972717
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,1023,0.06582880020141602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,1023,0.038104000687599185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,2047,0.12097280025482178
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,2047,0.065801602602005
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,float16,4095,0.22986719608306885
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,24,24,128,1,float16,fp8,4095,0.12082879543304444
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,1,0.00836160033941269
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,1,0.009220799803733826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,3,0.008484800159931184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,3,0.00846880003809929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,7,0.00897279977798462
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,7,0.008422400057315826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,15,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,15,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,31,0.008420799672603608
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,24,24,128,1,float16,fp8,2047,0.11947360038757324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,31,0.008387199789285659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,63,0.009486400336027146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,127,0.008367999643087386
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,63,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,127,0.008532799780368805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,255,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,255,0.00841279998421669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,511,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,511,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,1023,0.01252640038728714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,1023,0.01061440035700798
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,2047,0.019543999433517457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,2047,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,float16,4095,0.037062400579452516
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,24,24,128,1,float16,fp8,4095,0.018667200207710268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,1,0.03709119856357575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,1,0.031052801012992858
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,3,0.037064000964164734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,3,0.030988800525665283
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,7,0.0371071994304657
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,7,0.030926400423049928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,15,0.035734400153160095
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,15,0.03109920024871826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,31,0.035139200091362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,31,0.030929601192474364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,63,0.03915359973907471
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,63,0.03094240128993988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,127,0.041172799468040464
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,127,0.03291040062904358
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,255,0.06364319920539856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,255,0.038812801241874695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,511,0.06389279961585999
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,511,0.11543680429458618
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,1023,0.21773760318756102
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,1023,0.11912640333175659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,1,0.06370559930801392
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,1,0.05343520045280457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,float16,2047,0.4246240139007568
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,24,24,128,1,float16,fp8,2047,0.2267375946044922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,3,0.0637008011341095
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,7,0.05362880229949951
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,3,0.05354239940643311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,7,0.06369919776916504
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,15,0.06421759724617004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,15,0.05358240008354187
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,31,0.0658896028995514
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,31,0.05349439978599548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,63,0.06641280055046081
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,63,0.055415999889373777
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,127,0.0713263988494873
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,127,0.057336002588272095
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,255,0.11587680578231811
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,255,0.06636959910392762
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,1,0.11903519630432129
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,float16,511,0.21903839111328124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,24,24,128,1,float16,fp8,511,0.11698559522628785
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,1,0.10001120567321778
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,3,0.10055999755859375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,3,0.1190000057220459
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,7,0.11897920370101929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,15,0.10064480304718018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,15,0.12106239795684814
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,31,0.1210368037223816
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,7,0.10084480047225952
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,31,0.10065759420394897
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,63,0.10140639543533325
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,63,0.12158559560775757
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,127,0.13035199642181397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,127,0.10488799810409546
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,1,0.01056160032749176
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,float16,255,0.22107360363006592
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,1,0.009828799962997436
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,24,24,128,1,float16,fp8,255,0.12328159809112549
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,3,0.010254400223493576
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,3,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,7,0.010494399815797806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,7,0.010190399736166
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,15,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,15,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,31,0.010471999645233154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,31,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,63,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,63,0.009851200133562088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,127,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,127,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,255,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,255,0.010099200159311294
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,511,0.010623999685049058
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,511,0.012503999471664428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,1023,0.018960000574588777
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,1023,0.013540799915790557
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,2047,0.04118880033493042
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,2047,0.020603199303150178
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,float16,4095,0.0715279996395111
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,24,24,128,1,float16,fp8,4095,0.04168320000171662
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,1,0.22758240699768068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,1,0.1924496054649353
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,3,0.229531192779541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,3,0.19077919721603392
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,7,0.2301487922668457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,7,0.19281920194625854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,15,0.22921440601348878
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,15,0.19279199838638306
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,31,0.2310415983200073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,31,0.19152159690856935
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,63,0.23215360641479493
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,63,0.19280799627304077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,float16,127,0.24770081043243408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,24,24,128,1,float16,fp8,127,0.1966655969619751
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,1,0.45105600357055664
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,1,0.37325758934020997
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,3,0.45092000961303713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,3,0.3766335964202881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,7,0.4481632232666016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,7,0.3763808012008667
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,15,0.4509744167327881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,15,0.3732703924179077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,31,0.4511903762817383
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,31,0.3753119945526123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,1,0.012435200065374375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,1,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,3,0.01239520013332367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,float16,63,0.45104160308837893
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,3,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,24,24,128,1,float16,fp8,63,0.3769439935684204
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,7,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,7,0.012382400035858155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,15,0.012409599870443344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,31,0.012443199753761292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,31,0.010527999699115753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,15,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,63,0.012508800625801087
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,63,0.010527999699115753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,127,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,127,0.010572800040245056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,255,0.012427199631929398
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,255,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,511,0.018118399381637573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,511,0.012681600451469422
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,1023,0.03778879940509796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,1023,0.018335999548435213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,2047,0.0642911970615387
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,2047,0.038020798563957216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,1,0.01669439971446991
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,float16,4095,0.11735999584197998
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,24,24,128,1,float16,fp8,4095,0.06590399742126465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,3,0.01781439930200577
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,7,0.016835199296474458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,15,0.016833600401878358
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,1,0.014643199741840363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,31,0.017628799378871917
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,31,0.014593599736690522
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,63,0.016603200137615202
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,63,0.014558400213718414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,127,0.01659359931945801
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,127,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,255,0.02712000012397766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,255,0.014603200554847717
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,511,0.04522719979286194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,511,0.02890239953994751
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,1023,0.08006560206413268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,1023,0.04570879936218262
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,2047,0.15001280307769777
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,2047,0.08476319909095764
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,4095,0.28909599781036377
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,4095,0.158297598361969
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,3,0.016572800278663636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,float16,8191,0.5699632167816162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,8191,0.3015216112136841
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,1,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,7,0.015750400722026825
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,3,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,1,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,7,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,7,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,15,0.010494399815797806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,15,0.010480000078678131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,31,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,31,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,63,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,63,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,127,0.010507199913263321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,127,0.010320000350475311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,255,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,255,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,511,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,511,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,1023,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,1023,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,2047,0.011454399675130844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,2047,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,4095,0.012529599666595458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,4095,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,float16,8191,0.014590400457382201
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,8191,0.012540799379348756
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,16,128,1,float16,fp8,3,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,1,0.008382400125265121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,1,0.010579200088977813
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,3,0.008577600121498108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,3,0.008451200276613235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,7,0.010521599650382995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,15,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,7,0.008529599756002426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,31,0.00915360003709793
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,31,0.008584000170230865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,15,0.008686400204896926
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,16,128,1,float16,fp8,15,0.014707200229167938
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,63,0.009703999757766724
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,63,0.008513599634170532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,255,0.008739200234413148
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,127,0.008459199965000153
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,127,0.008761599659919739
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,255,0.008617600053548813
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,511,0.00862400010228157
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,511,0.008444800227880477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,1023,0.008560000360012055
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,1023,0.010516799986362457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,2047,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,2047,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,4095,0.012620800733566284
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,float16,8191,0.02712000012397766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,4095,0.01053759977221489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,1,0.012665599584579468
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,16,128,1,float16,fp8,8191,0.014548799395561219
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,3,0.012582400441169738
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,1,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,3,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,7,0.012600000202655792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,15,0.012579199671745301
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,7,0.012385600060224534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,15,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,31,0.012521600723266602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,31,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,63,0.01250080019235611
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,63,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,127,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,127,0.012507200241088867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,255,0.012964800000190735
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,511,0.028191998600959778
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,255,0.012515200674533844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,511,0.014584000408649444
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,1023,0.028841599822044373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,1023,0.04737600088119507
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,2047,0.08279520273208618
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,2047,0.04711039960384369
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,4095,0.15447839498519897
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,4095,0.08435840010643006
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,1,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,1,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,fp8,8191,0.15746079683303832
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,3,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,16,128,1,float16,float16,8191,0.2984015941619873
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,3,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,7,0.009161599725484849
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,7,0.010283199697732925
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,15,0.008683200180530547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,31,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,15,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,31,0.008427199721336365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,63,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,63,0.0083856001496315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,127,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,127,0.008372800052165985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,255,0.008575999736785888
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,255,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,511,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,511,0.008387199789285659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,1023,0.010500799864530563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,2047,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,1023,0.010320000350475311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,2047,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,4095,0.028327998518943787
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,4095,0.014523200690746307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,float16,8191,0.04523360133171082
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,16,128,1,float16,fp8,8191,0.027356800436973572
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,1,0.026921600103378296
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,3,0.026846399903297423
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,1,0.022836799919605254
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,3,0.02285760045051575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,7,0.026950401067733765
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,7,0.022720000147819518
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,15,0.022836799919605254
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,15,0.02690559923648834
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,31,0.026825600862503053
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,31,0.0227743998169899
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,63,0.026950401067733765
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,63,0.022758400440216063
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,127,0.031108799576759338
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,127,0.024699200689792634
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,255,0.04563199877738953
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,255,0.028940799832344054
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,511,0.046321600675582886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,511,0.08024479746818543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,1023,0.14891200065612792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,1023,0.08140000104904174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,2047,0.2874543905258179
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,2047,0.15490880012512206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,1,0.045368000864982605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,1,0.03882879912853241
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,3,0.045311999320983884
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,float16,4095,0.5614272117614746
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,3,0.03920960128307342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,16,128,1,float16,fp8,4095,0.303276801109314
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,7,0.03914240002632141
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,7,0.045238399505615236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,15,0.03903680145740509
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,15,0.045363199710845944
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,31,0.045905598998069765
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,31,0.03905439972877502
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,63,0.04774079918861389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,63,0.039083200693130496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,127,0.05128960013389587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,127,0.041280001401901245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,255,0.0803264021873474
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,255,0.04868960082530975
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,511,0.14986079931259155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,511,0.08166239857673645
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,1,0.0821008026599884
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,float16,1023,0.2866879940032959
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,1,0.0698303997516632
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,16,128,1,float16,fp8,1023,0.1521407961845398
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,3,0.06993119716644287
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,3,0.08219680190086365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,7,0.08220959901809692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,15,0.0833407998085022
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,7,0.06982880234718322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,15,0.0698303997516632
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,31,0.08388320207595826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,31,0.06998080015182495
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,63,0.08526880145072938
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,63,0.06994400024414063
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,127,0.08936799764633178
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,127,0.0724399983882904
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,255,0.15040960311889648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,255,0.0875711977481842
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,1,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,1,0.0083856001496315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,3,0.008414400368928909
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,float16,511,0.28736319541931155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,16,128,1,float16,fp8,511,0.15221279859542847
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,3,0.008388800173997879
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,7,0.008937600255012512
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,15,0.008457600325345992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,7,0.008452799916267396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,31,0.008353599905967712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,15,0.008568000048398972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,31,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,63,0.008884800225496292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,63,0.008416000008583068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,127,0.009875199943780898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,127,0.008427199721336365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,255,0.008481600135564805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,255,0.008462399989366532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,511,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,511,0.009391999989748
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,1023,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,1023,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,2047,0.02715040147304535
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,2047,0.012727999687194824
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,4095,0.04516479969024658
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,4095,0.027169600129127502
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,float16,8191,0.0800544023513794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,16,128,1,float16,fp8,8191,0.04538399875164032
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,1,0.1557760000228882
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,1,0.13097599744796753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,3,0.1547119975090027
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,3,0.13130079507827758
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,7,0.1559424042701721
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,7,0.1311776041984558
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,15,0.15607039928436278
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,15,0.13110400438308717
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,31,0.15838719606399537
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,31,0.13118879795074462
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,63,0.15815839767456055
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,63,0.1316208004951477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,127,0.1685487985610962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,127,0.13335039615631103
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,float16,255,0.2910207986831665
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,16,128,1,float16,fp8,255,0.16591839790344237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,1,0.3023760080337524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,1,0.2537856101989746
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,3,0.3024159908294678
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,3,0.2540463924407959
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,7,0.3038127899169922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,7,0.25214719772338867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,15,0.3043071985244751
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,15,0.2542320013046265
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,31,0.30288798809051515
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,31,0.25411200523376465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,63,0.3058799982070923
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,63,0.2528111934661865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,1,0.010507199913263321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,float16,127,0.3246799945831299
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,1,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,3,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,3,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,7,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,16,128,1,float16,fp8,127,0.2578144073486328
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,7,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,15,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,31,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,15,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,31,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,63,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,63,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,127,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,127,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,255,0.010555200278759003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,255,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,511,0.012654399871826172
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,511,0.01058880016207695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,1023,0.028787198662757873
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,1023,0.014481599628925323
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,2047,0.0471231997013092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,2047,0.02884640097618103
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,4095,0.08050240278244018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,4095,0.045844799280166625
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,1,0.016505600512027742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,float16,8191,0.15300639867782592
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,16,128,1,float16,fp8,8191,0.0844655990600586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,3,0.01653439998626709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,7,0.01648160070180893
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,15,0.01666560024023056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,1,0.014534400403499603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,31,0.01650879979133606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,31,0.014681600034236908
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,63,0.016468800604343414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,63,0.014616000652313232
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,127,0.016516800224781036
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,127,0.014502400159835815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,255,0.018588800728321076
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,255,0.014416000247001648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,511,0.03928320109844208
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,511,0.019356800615787505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,1023,0.06570720076560974
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,1023,0.03853920102119446
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,2047,0.12029759883880616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,2047,0.06612160205841064
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,4095,0.22912800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,4095,0.12127039432525635
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,3,0.014494399726390838
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,float16,8191,0.4479184150695801
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,8191,0.23729441165924073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,1,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,3,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,3,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,1,0.010492800176143647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,7,0.014473600685596466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,7,0.010384000092744827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,15,0.010471999645233154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,7,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,15,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,31,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,31,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,63,0.01055999994277954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,127,0.01053600013256073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,63,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,127,0.010363200306892395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,255,0.010552000254392624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,255,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,511,0.01053439974784851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,1023,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,511,0.010305599868297577
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,1023,0.010552000254392624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,2047,0.010608000308275222
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,4095,0.0126351997256279
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,2047,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,4095,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,float16,8191,0.014567999541759491
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,12,12,128,1,float16,fp8,8191,0.012503999471664428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,1,0.010601600259542465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,1,0.010315199941396713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,3,0.010556799918413162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,3,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,7,0.010603199899196624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,7,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,15,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,15,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,31,0.010582400113344192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,31,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,63,0.01053600013256073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,63,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,127,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,127,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,255,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,255,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,511,0.010718400031328202
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,511,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,1023,0.01056160032749176
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,1023,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,2047,0.01265919953584671
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,2047,0.01122720018029213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,4095,0.014545600116252898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,4095,0.012363199889659882
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,float16,8191,0.018611200153827667
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,12,12,128,1,float16,fp8,8191,0.014417600631713868
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,1,0.01250240057706833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,1,0.010593599826097488
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,3,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,3,0.010521599650382995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,7,0.012443199753761292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,7,0.010515200346708298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,15,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,15,0.01050880029797554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,31,0.012435200065374375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,31,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,63,0.01053759977221489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,63,0.012438400089740754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,127,0.012520000338554382
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,127,0.01055999994277954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,255,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,255,0.010574399679899215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,511,0.01268640011548996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,511,0.01852799952030182
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,1023,0.0374096006155014
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,1023,0.01863040030002594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,2047,0.06380000114440917
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,2047,0.038073599338531494
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,4095,0.06480960249900818
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,4095,0.11661440134048462
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,12,12,128,1,float16,fp8,15,0.014567999541759491
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,1,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,fp8,8191,0.12321759462356567
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,12,12,128,1,float16,float16,8191,0.22235360145568847
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,1,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,3,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,7,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,7,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,3,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,15,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,15,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,31,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,31,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,63,0.010384000092744827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,63,0.01058719977736473
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,127,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,127,0.010294400155544281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,255,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,255,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,511,0.010678400099277497
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,511,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,1023,0.012515200674533844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,1023,0.01053759977221489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,2047,0.012664000689983367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,4095,0.018611200153827667
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,2047,0.012510399520397186
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,4095,0.01449120044708252
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,float16,8191,0.03897759914398193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,12,12,128,1,float16,fp8,8191,0.01865600049495697
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,1,0.022814400494098663
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,3,0.022697600722312927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,1,0.02070080041885376
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,3,0.020606400072574617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,7,0.02272160053253174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,7,0.020580799877643587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,15,0.022623999416828154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,15,0.018721599876880646
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,31,0.022779199481010436
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,31,0.01886879950761795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,63,0.018724800646305086
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,63,0.02279199957847595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,127,0.02502720057964325
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,127,0.01887039989233017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,255,0.02065120041370392
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,255,0.03715519905090332
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,511,0.06367679834365844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,511,0.03811039924621582
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,1023,0.11541600227355957
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,1023,0.06562079787254334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,2047,0.21963200569152833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,2047,0.11962720155715942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,1,0.037011200189590455
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,1,0.030943998694419862
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,float16,4095,0.431334400177002
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,3,0.036929601430892946
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,12,12,128,1,float16,fp8,4095,0.22999680042266846
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,7,0.035755199193954465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,3,0.030961599946022034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,15,0.03535360097885132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,7,0.03091999888420105
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,15,0.03091999888420105
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,31,0.035183998942375186
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,31,0.030953601002693176
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,63,0.03911679983139038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,63,0.031043198704719544
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,127,0.041345599293708804
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,127,0.03300800025463104
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,255,0.06375839710235595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,255,0.03901279866695404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,511,0.11509599685668945
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,511,0.0640447974205017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,1,0.0637615978717804
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,float16,1023,0.21815040111541747
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,12,12,128,1,float16,fp8,1023,0.118721604347229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,1,0.053547197580337526
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,3,0.053457599878311154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,7,0.06386719942092896
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,3,0.06378399729728698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,7,0.053539198637008664
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,15,0.063755202293396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,15,0.05363839864730835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,31,0.06621760129928589
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,63,0.06732479929924011
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,31,0.053513598442077634
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,63,0.05553119778633118
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,127,0.07130879759788514
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,127,0.05684319734573364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,255,0.11554080247879028
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,255,0.0679967999458313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,1,0.008403199911117553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,float16,511,0.21886239051818848
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,1,0.008376000076532364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,12,12,128,1,float16,fp8,511,0.11707359552383423
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,3,0.008387199789285659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,3,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,7,0.00843520015478134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,7,0.008430399745702744
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,15,0.008534400165081025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,15,0.008374399691820144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,31,0.00846880003809929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,31,0.008564800024032593
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,63,0.009911999851465226
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,63,0.008398400247097015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,127,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,127,0.00841120034456253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,255,0.008705600351095199
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,255,0.00843520015478134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,511,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,511,0.008457600325345992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,1023,0.012603199481964112
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,1023,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,2047,0.018649600446224213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,2047,0.012547199428081513
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,4095,0.037206399440765384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,4095,0.01865759938955307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,fp8,8191,0.036052799224853514
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,12,12,128,1,float16,float16,8191,0.06418240070343018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,1,0.11906880140304565
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,1,0.10056480169296264
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,3,0.11764800548553467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,3,0.10053759813308716
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,7,0.11899839639663697
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,7,0.1005295991897583
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,15,0.12106239795684814
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,15,0.10061119794845581
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,31,0.12112159729003906
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,31,0.10127840042114258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,63,0.12115999460220336
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,63,0.10069600343704224
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,127,0.12968640327453612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,127,0.10442080497741699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,fp8,255,0.12470240592956543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,12,12,128,1,float16,float16,255,0.2209791898727417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,1,0.22861120700836182
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,1,0.1933135986328125
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,3,0.19271839857101442
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,3,0.2282815933227539
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,7,0.23112480640411376
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,7,0.19294240474700927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,15,0.22939999103546144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,15,0.1936735987663269
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,31,0.2311840057373047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,63,0.23064639568328857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,31,0.19375840425491334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,63,0.19482879638671874
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,1,0.010862399637699128
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,1,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,float16,127,0.24621119499206542
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,3,0.010843200236558914
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,3,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,12,12,128,1,float16,fp8,127,0.1967120051383972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,7,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,7,0.010635200142860412
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,15,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,15,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,31,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,31,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,63,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,127,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,63,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,127,0.008563199639320373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,255,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,255,0.010332799702882766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,511,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,511,0.010592000186443329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,1023,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,1023,0.019120000302791595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,2047,0.041196799278259276
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,2047,0.01889120042324066
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,4095,0.07043520212173462
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,4095,0.04138079881668091
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,1,0.01257600039243698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,float16,8191,0.13089760541915893
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,12,12,128,1,float16,fp8,8191,0.0745360016822815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,3,0.012540799379348756
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,7,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,1,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,15,0.012627199292182922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,15,0.012428800016641617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,31,0.01263359934091568
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,31,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,63,0.012647999823093415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,63,0.012444800138473511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,127,0.012604799866676331
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,127,0.012489599734544754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,255,0.012558400630950928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,255,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,511,0.027430400252342224
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,511,0.014582400023937226
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,1023,0.04586080014705658
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,1023,0.028958401083946227
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,2047,0.0811855971813202
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,2047,0.04711520075798035
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,4095,0.15205440521240235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,4095,0.08479840159416199
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,8191,0.29477438926696775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,3,0.012510399520397186
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,8191,0.15879199504852295
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,1,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,float16,16383,0.5782032012939453
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,16383,0.31473441123962403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,3,0.010527999699115753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,7,0.01053759977221489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,1,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,15,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,15,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,31,0.010360000282526016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,31,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,63,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,3,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,63,0.010329599678516387
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,127,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,127,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,255,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,255,0.010334400087594986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,511,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,511,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,1023,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,1023,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,2047,0.010558400303125381
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,2047,0.01056319996714592
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,4095,0.012591999769210816
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,4095,0.012451200187206269
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,8191,0.014500799775123595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,8191,0.012622399628162384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,16383,0.014519999921321868
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,float16,16383,0.016502399742603303
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,1,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,1,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,3,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,3,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,7,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,7,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,15,0.010332799702882766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,8,128,1,float16,fp8,7,0.00844319984316826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,15,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,31,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,63,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,31,0.008568000048398972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,63,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,127,0.010363200306892395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,127,0.009230399876832962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,255,0.01032480001449585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,255,0.010337600111961364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,511,0.010489600151777268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,511,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,1023,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,2047,0.01109279990196228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,1023,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,2047,0.010567999631166457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,4095,0.012563200294971466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,4095,0.012520000338554382
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,8191,0.014724799990653991
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,8191,0.01310880035161972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,float16,16383,0.029291200637817382
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,8,128,1,float16,fp8,16383,0.016484799981117248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,1,0.010344000160694122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,1,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,3,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,3,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,7,0.010492800176143647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,7,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,15,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,15,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,8,128,1,float16,fp8,7,0.012408000230789185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,31,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,31,0.010363200306892395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,63,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,63,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,127,0.010523200035095215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,127,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,255,0.010328000038862228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,511,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,255,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,511,0.01183520033955574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,1023,0.026888000965118408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,1023,0.014478400349617004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,2047,0.0450655996799469
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,2047,0.02743679881095886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,4095,0.0800432026386261
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,4095,0.04705759882926941
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,8191,0.15001920461654664
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,8191,0.0833407998085022
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,1,0.009387200325727462
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,float16,16383,0.2890655994415283
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,1,0.008953599631786347
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,8,128,1,float16,fp8,16383,0.16136959791183472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,3,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,7,0.008470399677753449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,3,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,7,0.00846719965338707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,15,0.00846719965338707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,15,0.008448000252246856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,31,0.008588799834251403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,31,0.008367999643087386
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,63,0.008505599945783615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,63,0.008432000130414962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,127,0.008484800159931184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,127,0.008582399785518646
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,255,0.008556800335645676
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,255,0.00840959995985031
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,511,0.008462399989366532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,511,0.008457600325345992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,1023,0.01050880029797554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,1023,0.008382400125265121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,2047,0.010599999874830245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,2047,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,4095,0.012664000689983367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,4095,0.012121599912643433
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,8191,0.027350398898124694
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,8191,0.014428800344467163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,float16,16383,0.045547199249267575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,8,128,1,float16,fp8,16383,0.028308799862861632
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,1,0.016868799924850464
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,3,0.01666560024023056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,1,0.016531200706958772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,3,0.01645440012216568
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,7,0.01653759926557541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,7,0.016471999883651733
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,15,0.014531199634075165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,15,0.016840000450611115
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,31,0.017272000014781953
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,31,0.014555199444293976
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,63,0.01653279960155487
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,63,0.01701440066099167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,127,0.0166143998503685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,127,0.01494400054216385
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,255,0.027163198590278624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,255,0.01462240070104599
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,511,0.04529440104961395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,511,0.028915199637413024
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,1023,0.07944480180740357
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,1023,0.046140798926353456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,2047,0.0834447979927063
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,2047,0.14811999797821046
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,4095,0.2856384038925171
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,4095,0.1593775987625122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,1,0.026830399036407472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,1,0.022755199670791627
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,3,0.026815998554229736
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,float16,8191,0.5613759994506836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,3,0.022815999388694764
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,8,128,1,float16,fp8,8191,0.30742559432983396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,7,0.02279680073261261
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,15,0.022705599665641785
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,15,0.026771199703216553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,7,0.026833599805831908
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,31,0.026868799328804018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,31,0.022859199345111846
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,63,0.02690559923648834
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,63,0.022771200537681578
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,127,0.030232000350952148
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,127,0.022951999306678773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,255,0.04516479969024658
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,255,0.029731199145317078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,511,0.07947840094566345
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,511,0.04668959975242615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,1023,0.14819040298461914
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,1023,0.08176320195198059
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,1,0.04527199864387512
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,1,0.03910079896450043
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,fp8,2047,0.1537071943283081
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,3,0.0452672004699707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,8,128,1,float16,float16,2047,0.2860208034515381
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,3,0.03881439864635468
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,7,0.04525440037250519
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,7,0.039078399538993835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,15,0.04518879950046539
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,15,0.0390751987695694
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,31,0.03902080059051514
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,63,0.047260800004005434
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,31,0.04527359902858734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,63,0.03914720118045807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,127,0.04936479926109314
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,127,0.04126079976558685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,255,0.04932160079479218
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,255,0.08041120171546937
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,511,0.14917440414428712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,511,0.08120319843292237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,1,0.008716800063848496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,1,0.008408000320196151
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,float16,1023,0.2862767934799194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,8,128,1,float16,fp8,1023,0.15300960540771485
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,3,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,7,0.008505599945783615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,3,0.010342399775981902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,7,0.008528000116348267
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,15,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,15,0.010318399965763092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,31,0.00841279998421669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,31,0.009060800075531006
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,63,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,63,0.008488000184297562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,127,0.009419199824333192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,127,0.010307200253009796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,255,0.00841120034456253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,255,0.008414400368928909
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,511,0.010305599868297577
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,511,0.009505599737167358
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,1023,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,1023,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,2047,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,2047,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,4095,0.027118399739265442
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,4095,0.014534400403499603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,8191,0.045244801044464114
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,8191,0.02801760137081146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,float16,16383,0.07959200143814087
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,8,128,1,float16,fp8,16383,0.045561599731445315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,1,0.08216800093650818
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,3,0.08231040239334106
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,1,0.06987360119819641
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,3,0.06920480132102966
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,7,0.08214399814605713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,7,0.06986719965934754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,15,0.08222079873085023
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,31,0.08439520001411438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,15,0.0697983980178833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,31,0.06981920003890991
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,63,0.07138400077819824
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,63,0.08471360206604003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,127,0.09036800265312195
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,127,0.07224479913711548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,255,0.1500208020210266
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,255,0.0908191978931427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,1,0.15537439584732055
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,float16,511,0.28707520961761473
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,8,128,1,float16,fp8,511,0.15221439599990844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,1,0.12933599948883057
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,3,0.13127039670944213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,3,0.15591520071029663
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,7,0.15594559907913208
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,7,0.13127520084381103
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,15,0.15792319774627686
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,15,0.1307423949241638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,31,0.13128639459609986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,31,0.15815680027008056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,63,0.1579200029373169
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,63,0.13212000131607055
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,127,0.16856319904327394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,127,0.13485759496688843
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,1,0.00851840004324913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,1,0.008459199965000153
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,float16,255,0.29105439186096194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,3,0.008489599823951722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,8,128,1,float16,fp8,255,0.17123039960861205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,3,0.008720000088214875
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,7,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,7,0.008451200276613235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,15,0.00844319984316826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,15,0.008513599634170532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,31,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,63,0.00902400016784668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,31,0.00841120034456253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,63,0.008419200032949447
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,127,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,127,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,255,0.0084927998483181
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,255,0.008499199897050858
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,511,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,511,0.008497600257396699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,1023,0.01239520013332367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,1023,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,2047,0.02680639922618866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,2047,0.012571200728416443
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,4095,0.044177600741386415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,4095,0.026800000667572023
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,8191,0.07857760190963745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,8191,0.04533439874649048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,1,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,float16,16383,0.14790079593658448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,8,128,1,float16,fp8,16383,0.08104959726333619
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,3,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,7,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,1,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,15,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,15,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,31,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,31,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,63,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,63,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,127,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,127,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,255,0.01050880029797554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,255,0.01029599979519844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,511,0.012505599856376648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,511,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,1023,0.026892799139022826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,1023,0.01449279934167862
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,2047,0.04503040015697479
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,2047,0.027185600996017457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,4095,0.08005120158195496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,4095,0.04541760087013245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,8191,0.14978879690170288
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,8191,0.08180800080299377
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,16383,0.2874511957168579
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,3,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,32767,0.2961904048919678
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,float16,32767,0.563585615158081
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,1,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,3,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,7,0.010480000078678131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,1,0.008481600135564805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,7,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,7,0.008433599770069123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,15,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,3,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,31,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,15,0.008444800227880477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,63,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,31,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,63,0.008523199707269669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,127,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,255,0.010480000078678131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,127,0.010251200199127198
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,511,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,511,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,255,0.009435199946165086
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,1023,0.010468800365924836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,2047,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,1023,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,2047,0.010471999645233154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,4095,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,4095,0.010625600069761276
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,8191,0.014545600116252898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,8191,0.01363999992609024
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,16383,0.016590400040149687
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,16383,0.015193599462509155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,float16,32767,0.018651199340820313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,4,128,1,float16,fp8,32767,0.016523200273513793
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,1,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,3,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,3,0.008452799916267396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,7,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,7,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,15,0.01117440015077591
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,15,0.008500800281763077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,31,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,31,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,63,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,63,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,127,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,127,0.008475200086832047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,255,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,511,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,255,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,511,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,1023,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,1023,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,2047,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,4095,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,2047,0.010571199655532836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,4095,0.01249760016798973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,8191,0.014523200690746307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,16383,0.016499200463294984
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,8191,0.012532800436019897
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,16383,0.01451680064201355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,float16,32767,0.03092319965362549
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,1,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,4,128,1,float16,fp8,32767,0.01652639955282211
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,1,0.008329600095748901
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,3,0.008609600365161896
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,3,0.00835679993033409
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,7,0.00851840004324913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,7,0.008367999643087386
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,15,0.008867199718952178
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,15,0.008448000252246856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,31,0.009532800316810608
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,63,0.008643200248479843
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,31,0.00840959995985031
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,63,0.008475200086832047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,127,0.008473599702119828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,255,0.008462399989366532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,127,0.008507200330495835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,255,0.008392000198364257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,511,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,511,0.00942080020904541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,1023,0.012449599802494049
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,1023,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,2047,0.02704319953918457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,2047,0.012542399764060973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,4095,0.04440160095691681
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,4095,0.026952001452445983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,8191,0.04487839937210083
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,8191,0.0782480001449585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,16383,0.14818880558013917
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,4,128,1,float16,fp8,16383,0.15209439992904664
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,16383,0.07981759905815125
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,1,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,1,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,fp8,32767,0.1489408016204834
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,3,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,4,128,1,float16,float16,32767,0.28585760593414306
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,7,0.010492800176143647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,3,0.01037440001964569
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,15,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,15,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,7,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,31,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,63,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,31,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,127,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,63,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,127,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,255,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,511,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,255,0.010337600111961364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,511,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,1023,0.010558400303125381
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,1023,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,2047,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,2047,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,4095,0.012636800110340119
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,4095,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,8191,0.014716799557209014
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,8191,0.012641599774360657
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,16383,0.029422399401664735
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,16383,0.016607999801635742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,float16,32767,0.04729439914226532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,4,128,1,float16,fp8,32767,0.03091520071029663
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,1,0.01265760064125061
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,3,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,3,0.01239520013332367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,1,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,7,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,7,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,15,0.012636800110340119
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,15,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,31,0.012636800110340119
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,31,0.012377600371837615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,63,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,63,0.012495999783277511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,127,0.012646399438381195
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,127,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,255,0.012558400630950928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,255,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,511,0.02795040011405945
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,1023,0.046177598834037784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,1023,0.028886398673057555
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,511,0.014737600088119506
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,2047,0.047070398926734924
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,2047,0.08171679973602294
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,4095,0.15206079483032225
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,4095,0.08221920132637024
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,8191,0.1524224042892456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,8191,0.29473280906677246
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,float16,16383,0.5798831939697265
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,1,0.016838400065898894
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,4,128,1,float16,fp8,16383,0.29374239444732664
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,1,0.014547200500965118
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,3,0.016809600591659545
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,7,0.016603200137615202
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,3,0.016494399309158324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,15,0.016571199893951415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,7,0.014825600385665893
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,15,0.014752000570297241
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,31,0.016657599806785585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,63,0.01674720048904419
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,31,0.014739200472831726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,63,0.014611199498176575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,127,0.01674720048904419
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,255,0.02707839906215668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,127,0.01475519984960556
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,255,0.014772799611091614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,511,0.04466559886932373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,511,0.028324800729751586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,1023,0.07977439761161804
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,1023,0.0458976000547409
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,2047,0.08088319897651672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,2047,0.14805439710617066
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,float16,4095,0.28630719184875486
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,1,0.0270687997341156
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,4,128,1,float16,fp8,4095,0.150271999835968
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,3,0.026899200677871705
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,1,0.02284640073776245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,7,0.02279680073261261
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,7,0.026929599046707154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,3,0.02295520007610321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,15,0.026870399713516235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,31,0.02693760097026825
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,15,0.022899200022220612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,31,0.02276960015296936
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,63,0.026888000965118408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,127,0.030782398581504822
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,127,0.022963200509548188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,63,0.02290239930152893
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,255,0.04527679979801178
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,255,0.02946079969406128
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,511,0.08002399802207946
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,511,0.04575999975204468
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,1023,0.1478800058364868
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,1023,0.08165919780731201
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,1,0.008539199829101562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,float16,2047,0.2851855993270874
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,1,0.008516799658536911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,4,128,1,float16,fp8,2047,0.15035040378570558
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,3,0.008479999750852585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,3,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,15,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,7,0.0084927998483181
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,7,0.008534400165081025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,15,0.008452799916267396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,31,0.008606400340795517
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,31,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,63,0.008398400247097015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,127,0.008505599945783615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,63,0.0084927998483181
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,127,0.008417599648237229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,255,0.008523199707269669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,255,0.008432000130414962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,511,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,511,0.008535999804735184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,1023,0.00963039994239807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,1023,0.009705600142478944
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,2047,0.01051200032234192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,4095,0.012539200484752655
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,2047,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,4095,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,8191,0.027518400549888612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,8191,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,16383,0.0453904002904892
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,16383,0.02804799973964691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,fp8,32767,0.04582720100879669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,4,128,1,float16,float16,32767,0.07860959768295288
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,1,0.0453904002904892
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,1,0.03901279866695404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,3,0.045187199115753175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,3,0.03913759887218475
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,7,0.04526880085468292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,7,0.039083200693130496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,15,0.04524160027503967
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,15,0.03916159868240356
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,31,0.03919839859008789
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,31,0.045296001434326175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,63,0.04732959866523743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,63,0.03914240002632141
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,127,0.05078719854354859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,127,0.04131200015544891
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,255,0.08035039901733398
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,255,0.04911839962005615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,511,0.14878079891204835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,511,0.0821183979511261
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,1,0.0821183979511261
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,float16,1023,0.2853519916534424
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,1,0.06981760263442993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,4,128,1,float16,fp8,1023,0.15067679882049562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,3,0.06984480023384095
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,3,0.08216959834098816
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,7,0.06993920207023621
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,7,0.08260160088539123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,15,0.08222560286521911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,15,0.07010239958763123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,31,0.06988959908485412
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,31,0.08412160277366638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,63,0.0841584026813507
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,63,0.07190080285072327
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,127,0.09038879871368408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,127,0.072598397731781
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,255,0.15042239427566528
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,255,0.08843680024147034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,1,0.00950239971280098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,float16,511,0.286844801902771
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,1,0.008582399785518646
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,3,0.008428800106048583
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,4,128,1,float16,fp8,511,0.1526927947998047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,3,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,7,0.009622400254011154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,15,0.008454400300979614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,7,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,15,0.010308799892663955
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,31,0.008564800024032593
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,31,0.00960479974746704
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,63,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,63,0.008448000252246856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,127,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,127,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,255,0.008563199639320373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,255,0.009796799719333648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,511,0.008568000048398972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,511,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,1023,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,1023,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,2047,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,2047,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,4095,0.027371200919151305
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,4095,0.014441600441932679
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,8191,0.027935999631881713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,8191,0.04521760046482086
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,16383,0.07987040281295776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,16383,0.04531359970569611
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,1,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,float16,32767,0.1484112024307251
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,4,128,1,float16,fp8,32767,0.08066719770431519
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,3,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,7,0.008459199965000153
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,1,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,15,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,15,0.00835679993033409
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,31,0.00841279998421669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,31,0.00840959995985031
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,63,0.008379200100898742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,63,0.008369600027799606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,127,0.008881600201129913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,127,0.008619199693202972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,255,0.008387199789285659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,255,0.008436799794435502
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,511,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,511,0.008401600271463394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,1023,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,1023,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,2047,0.026796799898147584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,2047,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,4095,0.04337919950485229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,4095,0.027625599503517152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,8191,0.07820159792900086
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,8191,0.04525279998779297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,16383,0.08001120090484619
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,16383,0.14712320566177367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,32767,0.2837696075439453
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,3,0.008539199829101562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,float16,65535,0.5558000087738038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,65535,0.29131040573120115
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,1,0.009719999879598618
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,3,0.01029599979519844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,3,0.008417599648237229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,7,0.008403199911117553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,7,0.008414400368928909
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,1,0.008537600189447403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,7,0.008414400368928909
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,15,0.00843999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,15,0.00851840004324913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,31,0.008454400300979614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,31,0.008556800335645676
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,63,0.010531199723482132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,63,0.00846560001373291
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,127,0.00840959995985031
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,127,0.008550400286912918
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,255,0.010900799930095673
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,255,0.008401600271463394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,511,0.010499200224876404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,511,0.00849440023303032
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,1023,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,2047,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,1023,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,2047,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,4095,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,4095,0.010582400113344192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,8191,0.014019200205802917
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,16383,0.018644799292087556
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,8191,0.012649600207805634
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,16383,0.01759680062532425
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,32767,0.018772800266742707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,32767,0.018632000684738158
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,float16,65535,0.02078240066766739
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,2,128,1,float16,fp8,65535,0.018636800348758698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,1,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,1,0.008388800173997879
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,3,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,3,0.00841120034456253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,7,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,7,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,15,0.010345599800348281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,15,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,31,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,31,0.00966399982571602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,63,0.01037440001964569
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,63,0.008451200276613235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,127,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,127,0.008500800281763077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,255,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,255,0.008459199965000153
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,511,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,1023,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,511,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,1023,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,2047,0.010331200063228607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,2047,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,4095,0.01220960021018982
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,4095,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,8191,0.014619199931621552
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,8191,0.013070400059223174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,16383,0.016715200245380403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,16383,0.014596800506114959
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,32767,0.01873279958963394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,32767,0.016539199650287627
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,fp8,65535,0.018691200017929076
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,2,128,1,float16,float16,65535,0.03519200086593628
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,1,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,1,0.010164800286293029
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,3,0.00838399976491928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,3,0.008500800281763077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,7,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,7,0.008425600081682205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,15,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,31,0.01037440001964569
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,15,0.0084927998483181
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,31,0.00894559994339943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,63,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,63,0.009105599671602248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,127,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,127,0.009379199892282485
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,2,128,1,float16,fp8,32767,0.1493391990661621
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,255,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,255,0.009380800276994705
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,511,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,1023,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,511,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,2047,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,1023,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,2047,0.011931200325489045
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,4095,0.027014398574829103
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,4095,0.014459200203418732
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,8191,0.045377600193023684
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,8191,0.028244799375534056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,16383,0.07958080172538758
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,16383,0.04544639885425568
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,32767,0.14726560115814208
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,32767,0.08002880215644836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,1,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,1,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,float16,65535,0.2840447902679443
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,2,128,1,float16,fp8,65535,0.14875839948654174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,3,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,7,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,7,0.010248000174760819
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,3,0.01055999994277954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,15,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,15,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,31,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,31,0.010195200145244599
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,63,0.010582400113344192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,63,0.009556800127029419
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,127,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,127,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,255,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,255,0.009347199648618697
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,511,0.010545600205659866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,511,0.009256000071763993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,1023,0.010500799864530563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,1023,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,2047,0.01056319996714592
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,2047,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,4095,0.01265760064125061
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,4095,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,8191,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,8191,0.01266240030527115
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,16383,0.016598400473594666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,16383,0.014567999541759491
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,32767,0.016531200706958772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,32767,0.03133440017700195
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,float16,65535,0.04792320132255554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,2,128,1,float16,fp8,65535,0.032583999633789065
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,1,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,1,0.010371199995279311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,3,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,7,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,3,0.010499200224876404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,7,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,15,0.01066880002617836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,31,0.010492800176143647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,31,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,15,0.010599999874830245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,63,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,63,0.01032159999012947
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,127,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,127,0.010734400153160096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,255,0.010580799728631973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,255,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,511,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,511,0.0127920001745224
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,1023,0.026878398656845093
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,1023,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,2047,0.04538240134716034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,2047,0.027267199754714967
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,4095,0.0799839973449707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,4095,0.046059200167655946
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,8191,0.14810559749603272
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,8191,0.08158559799194336
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,16383,0.15167839527130128
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,16383,0.28679039478302004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,1,0.012639999389648438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,1,0.012590399384498597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,float16,32767,0.5627952098846436
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,3,0.012720000743865967
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,2,128,1,float16,fp8,32767,0.2939136028289795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,7,0.01284320056438446
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,3,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,7,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,15,0.012678399682044983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,15,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,31,0.012723200023174286
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,63,0.012643200159072877
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,63,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,31,0.012559999525547028
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,127,0.012583999335765839
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,127,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,255,0.013230399787425995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,255,0.01239679977297783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,511,0.027539199590682982
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,511,0.01451680064201355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,1023,0.028911998867988585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,1023,0.04551199972629547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,2047,0.08048319816589355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,2047,0.047332799434661864
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,4095,0.15180799961090088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,4095,0.08203520178794861
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,1,0.016604800522327424
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,float16,8191,0.291974401473999
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,1,0.015281599760055543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,2,128,1,float16,fp8,8191,0.15227999687194824
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,3,0.016233600676059723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,3,0.01663520038127899
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,7,0.016545599699020384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,15,0.016568000614643096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,7,0.016680000722408293
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,15,0.01658879965543747
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,31,0.016582399606704712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,31,0.016577599942684172
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,63,0.01669919937849045
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,63,0.01656000018119812
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,127,0.01679680049419403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,127,0.01659200042486191
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,255,0.027350398898124694
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,255,0.016646400094032288
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,511,0.02874239981174469
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,511,0.04530239999294281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,1023,0.07929919958114624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,1023,0.04639039933681488
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,2047,0.1476655960083008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,2047,0.08126879930496216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,1,0.010515200346708298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,1,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,float16,4095,0.283404803276062
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,2,128,1,float16,fp8,4095,0.15223840475082398
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,3,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,3,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,7,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,7,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,15,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,15,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,31,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,31,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,63,0.010371199995279311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,127,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,63,0.008696000277996063
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,127,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,255,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,511,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,255,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,1023,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,1023,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,511,0.010339199751615524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,2047,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,2047,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,4095,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,4095,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,8191,0.014529600739479065
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,8191,0.013833600282669067
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,16383,0.02895520031452179
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,16383,0.016564799845218657
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,32767,0.04739519953727722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,32767,0.029230400919914246
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,float16,65535,0.08099679946899414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,1,0.026822400093078614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,1,0.024766400456428528
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,2,128,1,float16,fp8,65535,0.048470398783683775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,3,0.02680639922618866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,3,0.02468319982290268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,7,0.02704319953918457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,7,0.025227200984954835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,15,0.02677919864654541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,15,0.02470880001783371
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,31,0.026790401339530943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,31,0.024710400402545928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,63,0.024833600223064422
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,63,0.026932799816131593
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,127,0.029606398940086365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,127,0.024676799774169922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,255,0.029449599981307983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,255,0.04546400010585785
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,511,0.04636160135269165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,511,0.07963520288467407
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,1023,0.14772000312805175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,1023,0.08163679838180542
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,1,0.045270401239395144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,float16,2047,0.2841455936431885
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,1,0.040427199006080626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,2,128,1,float16,fp8,2047,0.15129920244216918
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,3,0.04054720103740692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,3,0.045316800475120544
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,7,0.04076800048351288
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,7,0.04530879855155945
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,15,0.04546400010585785
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,15,0.040612798929214475
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,31,0.04542079865932465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,31,0.04110879898071289
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,63,0.04678240120410919
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,63,0.03920960128307342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,127,0.050102400779724124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,127,0.04298880100250244
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,255,0.08014240264892578
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,255,0.050190401077270505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,511,0.1477455973625183
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,511,0.08336480259895325
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,1,0.009595199674367904
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,1,0.0084927998483181
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,float16,1023,0.2845599889755249
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,2,128,1,float16,fp8,1023,0.1517359972000122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,3,0.00957759991288185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,7,0.008561599999666214
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,7,0.008408000320196151
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,3,0.008420799672603608
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,15,0.008502399921417237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,15,0.0083856001496315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,31,0.008460800349712371
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,31,0.008515200018882752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,63,0.008398400247097015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,63,0.008382400125265121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,127,0.008510400354862214
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,127,0.008473599702119828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,255,0.008454400300979614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,255,0.008454400300979614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,511,0.00846880003809929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,511,0.008395200222730636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,1023,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,1023,0.00984639972448349
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,2047,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,4095,0.012507200241088867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,4095,0.011615999788045884
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,2047,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,8191,0.027854400873184203
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,8191,0.014545600116252898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,16383,0.04521439969539642
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,16383,0.028030401468276976
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,32767,0.07954559922218322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,32767,0.045747199654579164
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,float16,65535,0.14600160121917724
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,1,0.010283199697732925
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,2,128,1,float16,fp8,65535,0.07989119887351989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,3,0.010276799649000167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,7,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,15,0.010590399801731109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,1,0.008457600325345992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,31,0.008505599945783615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,31,0.008404800295829773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,63,0.008612799644470214
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,15,0.008529599756002426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,63,0.008817599713802337
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,127,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,127,0.009542399644851684
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,255,0.010361599922180175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,255,0.008718399703502655
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,511,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,511,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,1023,0.010521599650382995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,1023,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,2047,0.012505599856376648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,3,0.008416000008583068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,2047,0.010596799850463866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,4095,0.02707200050354004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,4095,0.014478400349617004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,8191,0.04529759883880615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,8191,0.027659198641777037
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,16383,0.07885439991950989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,16383,0.045731198787689206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,32767,0.08030239939689636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,32767,0.14695359468460084
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,65535,0.14836000204086303
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,65535,0.28404960632324217
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,131071,0.28820478916168213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,float16,131071,0.5569056034088135
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,1,0.00982080027461052
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,3,0.009656000137329101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,7,0.00873119980096817
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,1,1,128,1,float16,fp8,7,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,15,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,1,0.008441600203514098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,31,0.009537599980831146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,3,0.008580800145864487
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,31,0.008422400057315826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,63,0.008977600187063218
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,63,0.008457600325345992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,127,0.008590400218963623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,127,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,255,0.008523199707269669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,255,0.008367999643087386
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,511,0.00862400010228157
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,511,0.008380799740552902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,1023,0.008737599849700928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,1023,0.008401600271463394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,2047,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,2047,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,4095,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,4095,0.011043199896812439
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,8191,0.01250240057706833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,8191,0.012615999579429627
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,16383,0.015894399583339693
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,16383,0.01659359931945801
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,32767,0.016574400663375854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,32767,0.016711999475955964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,65535,0.018622399866580965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,65535,0.018628799915313722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,15,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,float16,131071,0.022683200240135194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,7,0.008542399853467941
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,1,1,128,1,float16,fp8,131071,0.020817600190639496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,1,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,1,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,3,0.009014400094747544
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,7,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,3,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,7,0.008430399745702744
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,15,0.009388799965381622
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,31,0.01032319962978363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,31,0.008403199911117553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,15,0.009308800101280212
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,63,0.008937600255012512
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,127,0.008460800349712371
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,127,0.00950080007314682
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,63,0.008577600121498108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,255,0.010078399628400802
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,255,0.008372800052165985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,511,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,511,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,1023,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,1023,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,2047,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,2047,0.010384000092744827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,4095,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,8191,0.012622399628162384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,4095,0.010548800230026245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,16383,0.018619200587272643
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,8191,0.012654399871826172
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,16383,0.016705599427223206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,32767,0.018620799481868743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,32767,0.018911999464035035
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,65535,0.020785599946975708
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,65535,0.0194815993309021
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,float16,131071,0.037966400384902954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,1,0.00854559987783432
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,1,1,128,1,float16,fp8,131071,0.022808000445365906
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,1,0.008523199707269669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,3,0.008379200100898742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,3,0.008551999926567078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,7,0.00854559987783432
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,7,0.00854720026254654
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,15,0.008486399799585343
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,15,0.008593600243330002
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,31,0.008433599770069123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,31,0.008497600257396699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,63,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,63,0.008379200100898742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,127,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,255,0.008507200330495835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,127,0.008947200328111648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,255,0.008508799970149994
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,511,0.008479999750852585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,511,0.008488000184297562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,1023,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,2047,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,1023,0.008884800225496292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,2047,0.010526400059461594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,4095,0.01170239970088005
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,4095,0.012542399764060973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,8191,0.014500799775123595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,8191,0.027425599098205567
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,16383,0.044950398802757266
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,16383,0.02813279926776886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,32767,0.045684799551963806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,32767,0.07943199872970581
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,65535,0.1467360019683838
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,65535,0.0804319977760315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,1,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,float16,131071,0.2837007999420166
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,1,1,128,1,float16,fp8,131071,0.1480479955673218
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,1,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,7,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,3,0.008460800349712371
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,3,0.009305600076913834
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,7,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,15,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,15,0.008473599702119828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,31,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,31,0.008416000008583068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,63,0.009161599725484849
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,63,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,127,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,127,0.008588799834251403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,255,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,255,0.00838399976491928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,511,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,1023,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,511,0.009908799827098847
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,2047,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,1023,0.008539199829101562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,2047,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,4095,0.010500799864530563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,4095,0.010499200224876404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,8191,0.014366400241851807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,8191,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,16383,0.016599999368190767
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,16383,0.01480640023946762
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,32767,0.018534399569034576
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,32767,0.016547200083732606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,65535,0.035097599029541016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,float16,131071,0.051614397764205934
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,65535,0.01866080015897751
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,1,1,128,1,float16,fp8,131071,0.035148799419403076
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,1,0.008380799740552902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,1,0.008401600271463394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,3,0.008539199829101562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,3,0.008398400247097015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,7,0.008560000360012055
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,7,0.008452799916267396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,15,0.008398400247097015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,15,0.008371199667453765
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,31,0.00859680026769638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,31,0.008363199979066848
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,63,0.00843520015478134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,63,0.008540800213813782
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,127,0.008363199979066848
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,127,0.008483199775218964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,255,0.009855999797582626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,255,0.008366400003433227
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,511,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,511,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,1023,0.012455999851226807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,1023,0.010556799918413162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,2047,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,2047,0.026793599128723145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,4095,0.02699199914932251
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,4095,0.04506880044937134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,8191,0.07831199765205384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,8191,0.04511840045452118
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,16383,0.14702399969100952
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,16383,0.079203200340271
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,32767,0.2831712007522583
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,32767,0.1478943943977356
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,1,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,1,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,3,0.010907199978828431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,float16,65535,0.556279993057251
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,3,0.010363200306892395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,7,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,1,1,128,1,float16,fp8,65535,0.28474400043487547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,15,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,7,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,31,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,15,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,31,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,63,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,63,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,127,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,127,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,255,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,255,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,1023,0.027059200406074523
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,511,0.012849600613117218
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,511,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,1023,0.014519999921321868
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,2047,0.028454399108886717
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,2047,0.04523360133171082
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,4095,0.07914559841156006
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,4095,0.04561119973659515
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,8191,0.14839040040969848
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,8191,0.08040159940719604
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,1,0.012520000338554382
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,float16,16383,0.2876575946807861
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,1,0.012462399899959564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,1,1,128,1,float16,fp8,16383,0.150654399394989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,3,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,3,0.012532800436019897
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,7,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,7,0.012617599964141846
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,15,0.01252640038728714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,15,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,31,0.01263200044631958
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,31,0.012521600723266602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,63,0.012559999525547028
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,63,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,127,0.012518399953842163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,127,0.01252480000257492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,255,0.012435200065374375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,255,0.012615999579429627
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,511,0.027347201108932497
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,511,0.014505599439144135
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,1023,0.04560959935188293
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,1023,0.028886398673057555
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,2047,0.08085439801216125
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,2047,0.047312000393867494
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,4095,0.15105439424514772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,4095,0.08227999806404114
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,1,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,1,0.010334400087594986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,float16,8191,0.29183039665222166
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,3,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,1,1,128,1,float16,fp8,8191,0.15111039876937865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,7,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,7,0.008395200222730636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,15,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,3,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,15,0.008451200276613235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,31,0.010316800326108932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,63,0.009297599643468856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,31,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,63,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,127,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,127,0.00891520008444786
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,255,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,255,0.008459199965000153
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,511,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,1023,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,511,0.010305599868297577
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,1023,0.009207999706268311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,2047,0.010543999820947647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,2047,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,4095,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,4095,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,8191,0.014473600685596466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,8191,0.01252799928188324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,16383,0.01658080071210861
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,16383,0.014595200121402741
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,32767,0.03091199994087219
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,32767,0.016655999422073364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,65535,0.04933600127696991
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,65535,0.032355201244354245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,float16,131071,0.08316640257835388
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,1,0.016551999747753142
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,1,1,128,1,float16,fp8,131071,0.04972479939460754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,1,0.016547200083732606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,3,0.01658719927072525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,7,0.016924799978733064
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,3,0.016663999855518342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,7,0.016572800278663636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,15,0.017209599912166595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,15,0.016649599373340606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,31,0.017003199458122252
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,31,0.016571199893951415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,63,0.016606399416923524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,63,0.01858399957418442
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,127,0.01658080071210861
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,127,0.01653600037097931
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,255,0.02746880054473877
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,255,0.016548800468444824
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,511,0.029054400324821473
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,511,0.04526079893112182
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,1023,0.07822080254554749
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,1023,0.04752320051193237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,2047,0.14741120338439942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,2047,0.08145440220832825
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,float16,4095,0.28356161117553713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,1,1,128,1,float16,fp8,4095,0.15040960311889648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,1,0.027140799164772033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,1,0.024854399263858795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,3,0.026940798759460448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,7,0.026836800575256347
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,3,0.025088000297546386
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,7,0.024902400374412537
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,15,0.026843199133872987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,31,0.026969599723815917
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,15,0.02502399981021881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,31,0.02479359954595566
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,63,0.02703840136528015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,127,0.02956480085849762
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,63,0.025089600682258607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,127,0.024935999512672426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,255,0.045256000757217404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,255,0.03091999888420105
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,511,0.08004800081253052
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,511,0.04752640128135681
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,1023,0.0828656017780304
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,1023,0.1479856014251709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,1,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,1,0.01034879982471466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,float16,2047,0.2838032007217407
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,3,0.010300800204277039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,1,1,128,1,float16,fp8,2047,0.15063680410385133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,7,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,3,0.009323199838399887
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,15,0.010489600151777268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,7,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,15,0.009644799679517747
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,31,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,31,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,63,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,127,0.010515200346708298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,63,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,127,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,255,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,511,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,255,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,511,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,1023,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,1023,0.01027199998497963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,2047,0.011587200313806533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,2047,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,4095,0.012583999335765839
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,8191,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,4095,0.01247519999742508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,8191,0.014300799369812012
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,16383,0.016156800091266632
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,16383,0.030079999566078187
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,32767,0.047521600127220155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,32767,0.03052479922771454
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,65535,0.08169440031051636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,65535,0.04790560007095337
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,float16,131071,0.1500175952911377
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,1,1,128,1,float16,fp8,131071,0.08251839876174927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,1,0.022710399329662324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,1,0.020870399475097657
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,3,0.022702400386333466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,3,0.022710399329662324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,7,0.022625599801540375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,7,0.0227183997631073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,15,0.022724799811840057
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,15,0.021723200380802155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,31,0.022707200050354003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,31,0.021040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,63,0.022694399952888487
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,63,0.022575999796390533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,127,0.022668799757957457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,127,0.02210720032453537
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,255,0.020635199546813966
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,255,0.022711999714374542
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,511,0.018718400597572328
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,511,0.01857440024614334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,1023,0.020751999318599702
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,float16,2047,0.024721600115299225
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,1023,0.020576000213623047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,1,128,1,float16,fp8,2047,0.022654399275779724
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,1,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,1,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,3,0.010516799986362457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,3,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,7,0.01051200032234192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,15,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,7,0.010305599868297577
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,15,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,31,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,31,0.010489600151777268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,63,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,63,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,127,0.010480000078678131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,127,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,255,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,511,0.012408000230789185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,255,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,511,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,1023,0.012531200051307678
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,1023,0.014678399264812469
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,float16,2047,0.029054400324821473
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,2,128,1,float16,fp8,2047,0.018503999710083006
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,1,0.01395999938249588
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,1,0.012529599666595458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,3,0.013916799426078796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,3,0.012582400441169738
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,7,0.014526399970054626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,7,0.012559999525547028
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,15,0.012611199915409089
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,15,0.012559999525547028
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,31,0.014244799315929414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,31,0.012598399817943574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,63,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,63,0.012534399330615998
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,127,0.01451680064201355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,255,0.014417600631713868
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,127,0.012547199428081513
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,255,0.012619200348854064
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,511,0.016582399606704712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,511,0.014590400457382201
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,1023,0.018614399433135986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,1023,0.030878400802612303
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,float16,2047,0.04842239916324616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,4,128,1,float16,fp8,2047,0.03219200074672699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,1,0.01480800062417984
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,1,0.012548799812793731
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,3,0.014561599493026734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,3,0.014472000300884247
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,7,0.014548799395561219
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,7,0.01249760016798973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,15,0.01467519998550415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,15,0.012548799812793731
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,31,0.014510400593280792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,31,0.014427199959754944
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,63,0.014531199634075165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,63,0.012495999783277511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,255,0.014560000598430633
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,127,0.014815999567508698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,127,0.01250080019235611
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,255,0.012539200484752655
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,511,0.029153600335121155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,511,0.01672160029411316
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,1023,0.04730400145053863
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,1023,0.031046399474143983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,float16,2047,0.08223519921302795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,128,8,128,1,float16,fp8,2047,0.049476799368858335
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,1,0.020641599595546723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,1,0.020694400370121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,3,0.020670400559902193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,3,0.02069920003414154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,7,0.020712000131607056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,7,0.020628799498081208
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,15,0.020771199464797975
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,15,0.02067199945449829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,31,0.020692799985408784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,31,0.020632000267505647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,63,0.020691199600696562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,63,0.020710399746894835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,127,0.02062080055475235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,255,0.020670400559902193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,127,0.02070239931344986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,255,0.018750399351119995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,511,0.01658399999141693
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,511,0.01658879965543747
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,1023,0.01449120044708252
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,1023,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,float16,2047,0.014529600739479065
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,1,128,1,float16,fp8,2047,0.014529600739479065
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,1,0.014548799395561219
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,1,0.014510400593280792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,3,0.014659200608730317
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,3,0.014550399780273438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,7,0.014532800018787383
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,7,0.014744000136852264
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,15,0.014531199634075165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,15,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,31,0.014552000164985656
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,31,0.01464959979057312
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,63,0.014601600170135499
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,63,0.014595200121402741
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,127,0.014638400077819825
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,255,0.014502400159835815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,127,0.014532800018787383
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,255,0.014696000516414643
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,511,0.012559999525547028
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,511,0.01252799928188324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,1023,0.012513600289821625
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,1023,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,float16,2047,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,2,128,1,float16,fp8,2047,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,1,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,3,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,1,0.011884800344705581
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,3,0.010710400342941285
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,7,0.011667200177907944
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,7,0.012571200728416443
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,15,0.011396799981594086
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,15,0.012489599734544754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,31,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,31,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,63,0.011604800075292587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,63,0.012371200323104858
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,127,0.01244639977812767
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,127,0.012531200051307678
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,255,0.012163200229406358
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,255,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,511,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,511,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,1023,0.010492800176143647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,1023,0.010489600151777268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,float16,2047,0.012544000148773193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,4,128,1,float16,fp8,2047,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,1,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,3,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,1,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,7,0.010993599891662598
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,3,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,7,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,15,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,15,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,31,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,31,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,63,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,127,0.011158400028944016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,63,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,127,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,255,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,255,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,511,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,511,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,1023,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,1023,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,float16,2047,0.012415999919176102
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,1,0.020683200657367708
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,1,0.020681600272655486
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,128,8,128,1,float16,fp8,2047,0.012572799623012543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,3,0.02067359983921051
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,3,0.02064319998025894
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,7,0.020656000077724456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,7,0.020895999670028687
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,15,0.020694400370121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,15,0.02064639925956726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,31,0.02064799964427948
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,31,0.021065600216388702
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,63,0.02078240066766739
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,63,0.02072319984436035
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,127,0.02065120041370392
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,127,0.02102400064468384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,255,0.020715199410915375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,255,0.020233599841594695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,511,0.018811200559139252
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,511,0.016596800088882445
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,1023,0.014550399780273438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,float16,2047,0.014548799395561219
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,1023,0.014820800721645355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,1,128,1,float16,fp8,2047,0.014547200500965118
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,1,0.014608000218868256
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,1,0.014710399508476257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,3,0.014604799449443817
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,3,0.014519999921321868
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,7,0.016502399742603303
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,7,0.014534400403499603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,15,0.014632000029087067
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,31,0.014596800506114959
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,15,0.014660799503326416
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,31,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,63,0.014534400403499603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,63,0.014654399454593658
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,127,0.014531199634075165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,127,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,255,0.014619199931621552
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,511,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,255,0.014467200636863709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,1023,0.012430399656295776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,511,0.012638400495052337
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,1023,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,float16,2047,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,1,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,2,128,1,float16,fp8,2047,0.01239520013332367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,1,0.01234399974346161
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,3,0.01247519999742508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,7,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,3,0.012505599856376648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,7,0.01173280030488968
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,15,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,15,0.012449599802494049
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,31,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,31,0.011449600011110306
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,63,0.01255040019750595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,63,0.012457600235939026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,127,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,127,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,255,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,255,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,511,0.012412799894809723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,1023,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,511,0.011830399930477142
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,1023,0.010518400371074677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,float16,2047,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,4,128,1,float16,fp8,2047,0.01239679977297783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,1,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,3,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,1,0.010592000186443329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,3,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,7,0.010556799918413162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,7,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,15,0.010553599894046783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,15,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,31,0.010691200196743012
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,63,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,31,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,63,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,127,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,127,0.010507199913263321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,255,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,511,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,255,0.01053439974784851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,511,0.010499200224876404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,1023,0.011059200018644333
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,1023,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,float16,2047,0.012558400630950928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,128,8,128,1,float16,fp8,2047,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,1,0.022716799378395082
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,1,0.02072640061378479
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,3,0.022702400386333466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,3,0.02067999988794327
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,7,0.02268799990415573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,7,0.02067359983921051
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,15,0.02263039946556091
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,15,0.020772799849510193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,31,0.02274080067873001
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,31,0.020670400559902193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,63,0.02231519967317581
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,63,0.020710399746894835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,127,0.020689600706100465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,127,0.02080480009317398
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,255,0.02064799964427948
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,255,0.02083359956741333
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,511,0.018614399433135986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,511,0.01855199933052063
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,1023,0.016697600483894348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,1023,0.015652799606323244
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,float16,2047,0.01860480010509491
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,1,128,1,float16,fp8,2047,0.016734400391578676
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,1,0.016551999747753142
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,3,0.016633599996566772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,1,0.016553600132465363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,3,0.016470399498939515
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,7,0.016659200191497803
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,7,0.016495999693870545
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,15,0.016628800332546233
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,31,0.01660960018634796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,15,0.016732800006866454
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,31,0.01650879979133606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,63,0.016542400419712066
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,63,0.016510400176048278
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,127,0.01658719927072525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,127,0.01655679941177368
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,255,0.016575999557971954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,511,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,255,0.016439999639987945
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,511,0.01454080045223236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,1023,0.01650879979133606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,1023,0.015729600191116334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,fp8,2047,0.01663679927587509
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,2,128,1,float16,float16,2047,0.018799999356269838
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,1,0.010548800230026245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,1,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,3,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,7,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,3,0.010576000064611435
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,7,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,15,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,15,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,31,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,31,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,63,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,63,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,127,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,127,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,255,0.010567999631166457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,255,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,511,0.012579199671745301
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,511,0.012408000230789185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,1023,0.014561599493026734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,1023,0.012638400495052337
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,float16,2047,0.02900640070438385
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,4,128,1,float16,fp8,2047,0.01658879965543747
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,1,0.01170400008559227
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,1,0.010620799660682679
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,3,0.010636799782514573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,3,0.010569600015878677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,7,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,7,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,15,0.012942400574684144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,15,0.010555200278759003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,31,0.01072160005569458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,31,0.010543999820947647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,63,0.011134400218725204
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,63,0.010496000200510025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,127,0.011102399975061416
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,127,0.010555200278759003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,255,0.010518400371074677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,511,0.014484800398349762
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,255,0.010582400113344192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,511,0.012535999715328216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,1023,0.02858240008354187
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,1023,0.01624799966812134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,float16,2047,0.045649600028991696
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,128,8,128,1,float16,fp8,2047,0.02954080104827881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,1,0.020838400721549986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,1,0.020662400126457214
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,3,0.020667199790477753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,7,0.020670400559902193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,3,0.020734399557113647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,15,0.020703999698162077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,7,0.020703999698162077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,15,0.020744000375270844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,31,0.020747199654579163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,31,0.020667199790477753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,63,0.020851199328899384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,63,0.020712000131607056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,127,0.020667199790477753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,127,0.020791999995708466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,255,0.020638400316238405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,255,0.020683200657367708
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,511,0.01850239932537079
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,1023,0.014609600603580474
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,1023,0.014595200121402741
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,511,0.016590400040149687
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,float16,2047,0.014547200500965118
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,1,128,1,float16,fp8,2047,0.014654399454593658
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,1,0.014617599546909332
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,1,0.014655999839305878
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,3,0.016579200327396394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,3,0.014609600603580474
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,7,0.014537599682807923
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,7,0.014764800667762756
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,15,0.014596800506114959
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,15,0.014611199498176575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,31,0.014555199444293976
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,31,0.015304000675678253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,63,0.014504000544548035
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,63,0.014500799775123595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,127,0.015548799932003022
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,255,0.014550399780273438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,255,0.014628799259662628
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,127,0.014548799395561219
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,511,0.014499199390411378
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,1023,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,1023,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,511,0.012724800407886505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,float16,2047,0.01250080019235611
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,2,128,1,float16,fp8,2047,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,1,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,1,0.012438400089740754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,3,0.012443199753761292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,3,0.012470400333404541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,7,0.012428800016641617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,7,0.012443199753761292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,15,0.01239359974861145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,15,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,31,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,31,0.012411200255155564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,63,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,63,0.012414400279521943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,127,0.012548799812793731
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,127,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,255,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,255,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,511,0.011611200124025344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,511,0.01141280010342598
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,1023,0.012438400089740754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,1023,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,float16,2047,0.01252799928188324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,4,128,1,float16,fp8,2047,0.012667199969291687
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,1,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,1,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,3,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,3,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,7,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,7,0.01034879982471466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,15,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,15,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,31,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,31,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,63,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,63,0.010529600083827972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,127,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,127,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,255,0.010332799702882766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,255,0.010332799702882766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,511,0.009384000301361084
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,511,0.010315199941396713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,1023,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,1023,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,float16,2047,0.011086399853229522
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,128,8,128,1,float16,fp8,2047,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,1,0.012380799651145935
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,1,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,3,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,3,0.012401600182056428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,7,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,7,0.012415999919176102
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,15,0.012491200119256973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,15,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,31,0.012470400333404541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,31,0.012444800138473511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,63,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,127,0.012455999851226807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,63,0.012462399899959564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,255,0.012411200255155564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,127,0.01247519999742508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,255,0.011779200285673141
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,511,0.01451680064201355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,float16,1023,0.016846400499343873
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,511,0.012598399817943574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,1,128,1,float16,fp8,1023,0.0165583997964859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,1,0.0126351997256279
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,1,0.01247519999742508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,3,0.012603199481964112
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,3,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,7,0.012555199861526489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,7,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,15,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,15,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,31,0.012985600531101227
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,31,0.01247519999742508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,63,0.012775999307632447
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,63,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,127,0.013524800539016724
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,127,0.012516799569129943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,255,0.012670400738716125
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,511,0.016686399281024934
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,255,0.012675200402736665
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,511,0.014591999351978302
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,float16,1023,0.030961599946022034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,1,0.01936960071325302
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,1,0.018377600610256194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,2,128,1,float16,fp8,1023,0.01897920072078705
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,3,0.019363200664520262
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,3,0.016606399416923524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,7,0.01940000057220459
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,7,0.018537600338459016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,15,0.018739199638366698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,31,0.018764799833297728
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,15,0.018671999871730804
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,31,0.01659359931945801
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,63,0.018753600120544434
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,127,0.018670399487018586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,63,0.018695999681949616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,255,0.018756799399852753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,127,0.01701119989156723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,511,0.0325408011674881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,255,0.01876319944858551
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,511,0.022724799811840057
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,1,0.022649599611759184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,float16,1023,0.05172160267829895
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,4,128,1,float16,fp8,1023,0.03341760039329529
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,1,0.018456000089645385
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,3,0.02272319942712784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,7,0.018603199720382692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,7,0.022654399275779724
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,3,0.018771199882030486
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,15,0.02274720072746277
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,15,0.018572799861431122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,31,0.022841599583625794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,31,0.018632000684738158
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,63,0.022652800381183624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,63,0.018700799345970152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,127,0.022731199860572815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,127,0.01855199933052063
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,255,0.018632000684738158
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,255,0.029359999299049377
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,511,0.04737919867038727
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,511,0.029684799909591674
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,1,0.016601599752902985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,fp8,1023,0.04933440089225769
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,1,0.014588800072669984
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,128,8,128,1,float16,float16,1023,0.08272320032119751
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,3,0.01454080045223236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,3,0.0165120005607605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,7,0.01687680035829544
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,15,0.016612799465656282
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,7,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,31,0.01650879979133606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,15,0.016155199706554414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,63,0.016497600078582763
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,63,0.014790399372577668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,31,0.014612799882888794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,127,0.01652960032224655
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,float16,255,0.01658399999141693
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,127,0.014735999703407287
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,1,128,1,float16,fp8,255,0.014591999351978302
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,1,0.01754560023546219
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,3,0.01857919991016388
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,1,0.01666560024023056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,3,0.016516800224781036
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,7,0.016568000614643096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,7,0.018724800646305086
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,15,0.017664000391960144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,15,0.016625599563121797
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,31,0.018593600392341612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,31,0.016631999611854555
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,63,0.018636800348758698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,127,0.018596799671649934
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,63,0.016646400094032288
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,127,0.016889600455760954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,float16,255,0.018670399487018586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,2,128,1,float16,fp8,255,0.016638399660587312
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,1,0.026921600103378296
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,1,0.031086400151252747
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,3,0.03088639974594116
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,3,0.02682879865169525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,7,0.030737599730491637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,7,0.027403199672698976
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,15,0.030750399827957152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,15,0.02720159888267517
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,31,0.030929601192474364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,31,0.026795199513435362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,63,0.030881598591804504
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,63,0.027372801303863527
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,127,0.030926400423049928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,float16,255,0.0347680002450943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,127,0.026822400093078614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,4,128,1,float16,fp8,255,0.0288239985704422
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,1,0.02687999904155731
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,3,0.03705120086669922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,1,0.036977601051330564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,3,0.026841598749160766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,7,0.02693440020084381
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,7,0.037067198753356935
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,15,0.037027201056480406
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,15,0.026956799626350402
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,31,0.03707360029220581
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,31,0.026817598938941957
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,63,0.02682879865169525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,63,0.03720319867134094
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,127,0.03917120099067688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,127,0.02714560031890869
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,float16,255,0.048291200399398805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,1,0.024796800315380098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,128,8,128,1,float16,fp8,255,0.03112640082836151
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,1,0.022606399655342103
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,3,0.024196800589561463
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,7,0.024691200256347655
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,3,0.022654399275779724
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,7,0.021166400611400606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,15,0.02486239969730377
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,15,0.022679999470710754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,31,0.023548799753189086
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,31,0.022657600045204163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,63,0.02280000001192093
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,63,0.020904000103473663
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,float16,127,0.024779200553894043
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,1,128,1,float16,fp8,127,0.02107519954442978
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,1,0.025812798738479616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,1,0.022993600368499754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,3,0.02560800015926361
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,3,0.022716799378395082
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,7,0.026947200298309326
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,7,0.022811199724674224
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,15,0.025187200307846068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,15,0.02292799949645996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,31,0.026815998554229736
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,31,0.022707200050354003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,63,0.026785600185394286
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,63,0.022735999524593355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,float16,127,0.027470400929450987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,2,128,1,float16,fp8,127,0.02290560007095337
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,1,0.051609599590301515
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,1,0.0472815990447998
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,3,0.051470398902893066
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,3,0.047251200675964354
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,7,0.051894402503967284
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,15,0.0514303982257843
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,7,0.047295999526977536
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,15,0.0473008006811142
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,31,0.05264480113983154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,31,0.04728800058364868
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,63,0.05523520112037659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,63,0.04725759923458099
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,fp8,127,0.04870879948139191
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,4,128,1,float16,float16,127,0.058156800270080564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,1,0.06573439836502075
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,1,0.045351999998092654
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,3,0.06571360230445862
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,3,0.045296001434326175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,7,0.06580320000648499
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,7,0.04554400146007538
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,15,0.04522719979286194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,15,0.06571040153503419
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,31,0.06679199934005738
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,31,0.045212799310684205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,63,0.06590240001678467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,63,0.04737440049648285
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,float16,127,0.0678879976272583
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,128,8,128,1,float16,fp8,127,0.047332799434661864
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,1,0.020735999941825865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,1,0.020632000267505647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,3,0.020665599405765532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,3,0.020670400559902193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,7,0.020688000321388244
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,7,0.02067680060863495
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,15,0.02075839936733246
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,31,0.020664000511169435
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,15,0.02067359983921051
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,31,0.02064799964427948
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,63,0.02070080041885376
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,127,0.020739200711250304
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,63,0.020660799741744996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,127,0.020729599893093108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,255,0.020579199492931365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,511,0.017084799706935883
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,255,0.020657600462436677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,1023,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,1023,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,511,0.018619200587272643
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,float16,2047,0.014604799449443817
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,1,128,1,float16,fp8,2047,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,1,0.014582400023937226
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,1,0.01682559996843338
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,3,0.014833599328994751
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,3,0.014556799829006196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,7,0.01454399973154068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,7,0.01671839952468872
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,15,0.01647839993238449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,15,0.016091200709342956
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,31,0.016761599481105803
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,63,0.016568000614643096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,31,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,127,0.016574400663375854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,63,0.016812799870967864
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,127,0.014593599736690522
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,255,0.016540800034999848
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,511,0.014558400213718414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,255,0.015107199549674988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,511,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,1023,0.012470400333404541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,float16,2047,0.012971200048923492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,1023,0.012585599720478059
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,2,128,1,float16,fp8,2047,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,1,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,3,0.011659199744462967
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,1,0.010795199871063232
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,3,0.010548800230026245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,7,0.012187200039625168
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,15,0.010507199913263321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,7,0.010764800012111664
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,15,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,31,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,63,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,31,0.012614400684833526
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,63,0.010897599905729295
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,127,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,127,0.011640000343322753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,255,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,255,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,1023,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,511,0.010644800215959548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,511,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,1023,0.010507199913263321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,float16,2047,0.012665599584579468
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,4,128,1,float16,fp8,2047,0.012436799705028534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,1,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,3,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,1,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,3,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,7,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,7,0.010311999917030334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,15,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,15,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,31,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,63,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,63,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,31,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,127,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,127,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,255,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,255,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,511,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,511,0.010892800241708755
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,1023,0.010521599650382995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,1023,0.01096320003271103
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,float16,2047,0.012564800679683685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,128,8,128,1,float16,fp8,2047,0.012495999783277511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,1,0.037006399035453795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,1,0.033004799485206605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,3,0.03705599904060364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,3,0.032950401306152344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,7,0.03707999885082245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,7,0.03295199871063233
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,15,0.03720479905605316
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,15,0.03293919861316681
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,31,0.037169599533081056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,31,0.03290719985961914
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,fp8,63,0.03296479880809784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,1,128,1,float16,float16,63,0.038910400867462155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,1,0.0391072005033493
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,1,0.04337440133094787
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,3,0.04325119853019714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,3,0.03909119963645935
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,7,0.04320479929447174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,7,0.0391072005033493
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,15,0.04370720088481903
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,15,0.039073601365089417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,31,0.04535680115222931
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,31,0.03917919993400574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,float16,63,0.04634400010108948
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,2,128,1,float16,fp8,63,0.03909280002117157
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,1,0.09675840139389039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,1,0.08590880036354065
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,3,0.09850559830665588
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,3,0.08598880171775818
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,7,0.09860320091247558
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,7,0.08591359853744507
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,15,0.09939680099487305
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,15,0.08622400164604187
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,31,0.10147520303726196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,float16,63,0.10347679853439332
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,31,0.08669440150260925
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,4,128,1,float16,fp8,63,0.08818879723548889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,1,0.12299840450286866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,1,0.08271359801292419
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,3,0.12309119701385499
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,3,0.08232160210609436
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,7,0.08349599838256835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,7,0.12350399494171142
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,15,0.12321439981460572
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,15,0.0842736005783081
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,31,0.08430240154266358
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,31,0.12388800382614136
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,float16,63,0.12468639612197877
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,128,8,128,1,float16,fp8,63,0.08444960117340088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,1,0.06881279945373535
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,1,0.05706560015678406
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,3,0.06824319958686828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,7,0.06787359714508057
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,7,0.057713598012924194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,3,0.05791040062904358
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,15,0.06786559820175171
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,15,0.05756639838218689
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,float16,31,0.06959840059280395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,1,128,1,float16,fp8,31,0.0575872004032135
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,1,0.08025760054588318
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,1,0.06963520050048828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,3,0.08031359910964966
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,3,0.0678991973400116
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,7,0.06872640252113342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,7,0.08165439963340759
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,15,0.08129760026931762
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,15,0.06828160285949707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,float16,31,0.08299840092658997
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,2,128,1,float16,fp8,31,0.07002720236778259
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,1,0.18892159461975097
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,3,0.1903807997703552
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,1,0.164193594455719
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,3,0.16463680267333985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,7,0.19070559740066528
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,7,0.1642959952354431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,15,0.19127199649810792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,15,0.16526080369949342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,float16,31,0.19397759437561035
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,4,128,1,float16,fp8,31,0.16527520418167113
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,1,0.23851358890533447
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,1,0.15801119804382324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,3,0.23932158946990967
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,3,0.1583888053894043
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,7,0.1584272027015686
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,7,0.23955199718475342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,15,0.23919360637664794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,15,0.1584496021270752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,1,0.020721599459648132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,3,0.02067999988794327
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,1,0.020681600272655486
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,float16,31,0.2392944097518921
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,128,8,128,1,float16,fp8,31,0.15834879875183105
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,7,0.021324799954891206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,3,0.020745599269866945
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,7,0.02078080028295517
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,15,0.020927999913692475
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,31,0.020761600136756896
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,15,0.020664000511169435
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,31,0.020919999480247496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,127,0.021689599752426146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,63,0.02067999988794327
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,63,0.02091359943151474
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,127,0.020798400044441223
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,255,0.02072319984436035
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,511,0.01870879977941513
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,511,0.016678400337696075
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,255,0.020659199357032774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,1023,0.014748799800872802
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,float16,2047,0.01653600037097931
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,2047,0.01464959979057312
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,1,128,1,float16,fp8,1023,0.014779199659824372
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,1,0.016519999504089354
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,1,0.015331199765205384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,3,0.016564799845218657
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,3,0.014550399780273438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,7,0.016510400176048278
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,7,0.01653759926557541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,15,0.016513599455356597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,15,0.014542399346828461
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,31,0.016497600078582763
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,31,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,63,0.01629440039396286
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,63,0.01642400026321411
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,127,0.016491200029850005
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,127,0.014691199362277984
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,255,0.016531200706958772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,255,0.014561599493026734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,511,0.014556799829006196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,511,0.014440000057220459
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,1023,0.01255200058221817
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,1023,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,fp8,2047,0.014468799531459808
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,1,0.012489599734544754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,2,128,1,float16,float16,2047,0.014732800424098969
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,1,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,3,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,3,0.01255040019750595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,7,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,7,0.011641599982976914
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,15,0.012564800679683685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,15,0.012438400089740754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,31,0.012518399953842163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,31,0.012415999919176102
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,63,0.012449599802494049
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,63,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,127,0.012385600060224534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,127,0.01090560033917427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,255,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,255,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,511,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,511,0.01055999994277954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,1023,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,1023,0.012577599287033081
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,float16,2047,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,1,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,4,128,1,float16,fp8,2047,0.014510400593280792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,1,0.0083856001496315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,3,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,3,0.008483199775218964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,7,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,7,0.008872000128030777
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,15,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,15,0.008449599891901017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,31,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,31,0.008454400300979614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,63,0.008479999750852585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,127,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,63,0.008918400108814239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,127,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,255,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,255,0.009337600320577621
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,511,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,1023,0.012547199428081513
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,1023,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,511,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,1,0.020657600462436677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,fp8,2047,0.014510400593280792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,128,8,128,1,float16,float16,2047,0.027139198780059815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,1,0.018673600256443025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,3,0.02064799964427948
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,3,0.018691200017929076
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,7,0.019275200366973878
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,7,0.01870400011539459
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,15,0.020558400452136992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,15,0.018713599443435668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,31,0.02067199945449829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,63,0.020688000321388244
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,31,0.018771199882030486
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,63,0.018692800402641298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,127,0.02003680020570755
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,127,0.01879040002822876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,255,0.01865759938955307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,255,0.018606400489807128
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,511,0.016697600483894348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,511,0.01658719927072525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,1023,0.01865279972553253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,1023,0.01866399943828583
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,float16,2047,0.02273920029401779
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,1,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,1,128,1,float16,fp8,2047,0.020695999264717102
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,1,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,3,0.010558400303125381
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,3,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,7,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,7,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,15,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,15,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,31,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,31,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,63,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,63,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,127,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,127,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,255,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,255,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,511,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,511,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,1023,0.012580800056457519
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,1023,0.014582400023937226
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,fp8,2047,0.01749439984560013
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,2,128,1,float16,float16,2047,0.029016000032424927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,1,0.012508800625801087
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,1,0.012515200674533844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,3,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,3,0.012508800625801087
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,7,0.012567999958992004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,7,0.012505599856376648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,15,0.012561599910259246
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,15,0.012531200051307678
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,31,0.014046399295330048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,31,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,63,0.013851200044155122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,63,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,127,0.014524799585342408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,255,0.01255040019750595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,127,0.012785600125789642
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,255,0.012505599856376648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,511,0.016543999314308167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,1023,0.02962239980697632
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,511,0.015095999836921692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,1023,0.01863519996404648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,1,0.014548799395561219
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,float16,2047,0.04745759963989258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,4,128,1,float16,fp8,2047,0.032678401470184325
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,3,0.014480000734329224
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,3,0.012627199292182922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,1,0.01255359947681427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,7,0.012683199346065521
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,7,0.014523200690746307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,15,0.014580799639225006
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,31,0.014593599736690522
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,15,0.012787200510501862
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,31,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,127,0.014628799259662628
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,63,0.014561599493026734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,63,0.012606400251388549
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,127,0.013128000497817992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,255,0.012694400548934937
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,255,0.01454080045223236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,511,0.028865599632263185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,511,0.016683200001716615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,1023,0.047305598855018616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,1023,0.02984800040721893
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,1,0.018646399676799773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,fp8,2047,0.049619200825691226
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,96,8,128,1,float16,float16,2047,0.08220800161361694
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,1,0.01860959976911545
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,3,0.018622399866580965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,3,0.018529599905014037
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,7,0.018539200723171233
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,7,0.018559999763965607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,15,0.01873279958963394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,15,0.018643200397491455
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,31,0.018624000251293182
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,31,0.018559999763965607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,63,0.018564799427986146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,63,0.018545599281787874
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,127,0.018622399866580965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,127,0.018580800294876097
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,255,0.018588800728321076
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,255,0.017927999794483184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,511,0.014574399590492249
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,511,0.014691199362277984
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,1023,0.014500799775123595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,1023,0.012508800625801087
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,float16,2047,0.014552000164985656
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,1,128,1,float16,fp8,2047,0.014451199769973755
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,1,0.013691200315952301
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,1,0.012548799812793731
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,3,0.014459200203418732
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,3,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,7,0.01454080045223236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,15,0.013147200644016265
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,7,0.0144896000623703
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,31,0.013700799643993377
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,31,0.014467200636863709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,15,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,63,0.013052800297737121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,63,0.01443679928779602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,127,0.013767999410629273
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,255,0.014460800588130951
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,255,0.012516799569129943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,127,0.01255359947681427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,511,0.012401600182056428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,511,0.012564800679683685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,1023,0.012433599680662155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,1023,0.012398400157690049
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,fp8,2047,0.012167999893426895
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,2,128,1,float16,float16,2047,0.012625600397586822
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,1,0.010516799986362457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,1,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,3,0.01106560006737709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,3,0.01051200032234192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,7,0.011116799712181092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,7,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,15,0.010596799850463866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,15,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,31,0.010689599812030793
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,31,0.010468800365924836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,63,0.012359999865293504
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,127,0.012411200255155564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,63,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,127,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,255,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,255,0.010726399719715118
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,511,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,1023,0.010513599961996078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,511,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,1023,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,float16,2047,0.012539200484752655
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,4,128,1,float16,fp8,2047,0.012308800220489502
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,1,0.010550399869680404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,3,0.010556799918413162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,3,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,1,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,7,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,7,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,15,0.010543999820947647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,15,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,31,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,31,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,63,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,63,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,127,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,255,0.01055999994277954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,127,0.010735999792814255
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,255,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,511,0.01053759977221489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,511,0.010713600367307664
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,1023,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,1023,0.010667199641466141
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,fp8,2047,0.010492800176143647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,1,0.018755200505256652
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,96,8,128,1,float16,float16,2047,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,3,0.018700799345970152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,1,0.01863040030002594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,3,0.01851679980754852
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,7,0.018769599497318268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,7,0.018561600148677825
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,15,0.01870400011539459
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,15,0.01868959963321686
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,31,0.018639999628067016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,31,0.018566399812698364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,63,0.018620799481868743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,63,0.01865759938955307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,127,0.018588800728321076
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,255,0.018590399622917177
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,127,0.018638400733470915
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,255,0.018505600094795228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,511,0.01658879965543747
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,511,0.016579200327396394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,1023,0.014691199362277984
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,1023,0.012558400630950928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,float16,2047,0.0147024005651474
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,1,128,1,float16,fp8,2047,0.014563199877738953
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,1,0.014523200690746307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,1,0.014401599764823914
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,3,0.014483200013637542
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,7,0.014486399292945863
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,3,0.013007999956607818
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,7,0.014299200475215912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,15,0.014496000111103058
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,15,0.01308799982070923
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,31,0.014550399780273438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,63,0.014619199931621552
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,31,0.014552000164985656
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,63,0.01427839994430542
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,127,0.014535999298095703
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,127,0.014523200690746307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,255,0.014382399618625641
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,511,0.012564800679683685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,255,0.012503999471664428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,511,0.012534399330615998
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,1023,0.012596799433231354
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,1023,0.01249760016798973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,float16,2047,0.01263200044631958
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,2,128,1,float16,fp8,2047,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,1,0.011739200353622437
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,1,0.01056319996714592
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,3,0.012383999675512314
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,3,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,7,0.01202239990234375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,7,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,15,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,31,0.012444800138473511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,15,0.01249760016798973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,31,0.010521599650382995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,63,0.012380799651145935
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,127,0.010492800176143647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,63,0.012385600060224534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,127,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,255,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,255,0.012388800084590913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,511,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,511,0.010532800108194351
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,1023,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,1023,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,float16,2047,0.012503999471664428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,4,128,1,float16,fp8,2047,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,1,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,1,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,3,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,3,0.01064160019159317
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,7,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,15,0.010480000078678131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,7,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,15,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,31,0.010603199899196624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,31,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,63,0.010518400371074677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,63,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,127,0.010526400059461594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,127,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,255,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,255,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,511,0.010611200332641601
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,511,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,1023,0.010532800108194351
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,1023,0.010523200035095215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,float16,2047,0.012495999783277511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,96,8,128,1,float16,fp8,2047,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,1,0.01860959976911545
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,1,0.018670399487018586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,3,0.01921440064907074
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,3,0.01868479996919632
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,7,0.01868479996919632
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,7,0.01884319931268692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,15,0.0186271995306015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,15,0.01863359957933426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,31,0.019827200472354888
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,31,0.018654400110244752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,63,0.018753600120544434
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,63,0.01866080015897751
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,127,0.018643200397491455
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,127,0.018636800348758698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,255,0.01879040002822876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,255,0.018632000684738158
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,511,0.016574400663375854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,511,0.016475200653076172
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,1023,0.014561599493026734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,1023,0.014529600739479065
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,float16,2047,0.016707199811935424
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,1,0.014550399780273438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,1,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,1,128,1,float16,fp8,2047,0.01658399999141693
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,3,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,7,0.014574399590492249
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,7,0.014526399970054626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,3,0.01475680023431778
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,15,0.014643199741840363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,31,0.014803199470043183
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,15,0.014449599385261535
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,31,0.01451359987258911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,63,0.01475519984960556
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,127,0.014612799882888794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,63,0.014510400593280792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,127,0.014640000462532044
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,255,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,255,0.014563199877738953
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,511,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,511,0.014419199526309967
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,1023,0.014788800477981567
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,2047,0.01655520051717758
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,float16,2047,0.018505600094795228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,2,128,1,float16,fp8,1023,0.014567999541759491
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,1,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,3,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,1,0.010627199709415436
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,3,0.010360000282526016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,7,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,7,0.010566399991512298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,15,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,15,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,31,0.010531199723482132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,63,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,31,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,63,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,127,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,127,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,255,0.01058719977736473
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,255,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,511,0.012464000284671784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,511,0.010468800365924836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,1023,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,1023,0.014590400457382201
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,float16,2047,0.029104000329971312
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,1,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,4,128,1,float16,fp8,2047,0.016633599996566772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,1,0.010529600083827972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,3,0.012382400035858155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,3,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,7,0.012392000108957291
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,7,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,15,0.012366399914026261
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,15,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,31,0.012342400103807449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,31,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,63,0.012191999703645706
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,63,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,127,0.012291199713945388
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,127,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,255,0.011769600212574005
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,255,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,511,0.014398400485515595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,511,0.012444800138473511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,1023,0.028380799293518066
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,1023,0.014769600331783294
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,float16,2047,0.045614400506019594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,96,8,128,1,float16,fp8,2047,0.02885119915008545
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,1,0.01870879977941513
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,1,0.0186271995306015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,3,0.018777599930763243
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,3,0.018636800348758698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,7,0.01870879977941513
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,7,0.018615999817848207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,31,0.018662400543689728
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,15,0.01857440024614334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,15,0.019043199717998505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,31,0.018643200397491455
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,63,0.018646399676799773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,63,0.018743999302387238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,127,0.018624000251293182
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,127,0.018588800728321076
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,255,0.018611200153827667
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,255,0.016628800332546233
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,511,0.016230399906635284
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,511,0.016476799547672272
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,1023,0.014548799395561219
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,1023,0.01454080045223236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,float16,2047,0.014657600224018097
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,1,128,1,float16,fp8,2047,0.014849600195884705
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,1,0.01441120058298111
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,1,0.014587199687957764
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,3,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,7,0.014542399346828461
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,3,0.014561599493026734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,7,0.014484800398349762
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,15,0.014459200203418732
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,15,0.01451359987258911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,31,0.014548799395561219
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,31,0.01446560025215149
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,63,0.014510400593280792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,63,0.013979199528694152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,127,0.014430400729179383
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,127,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,255,0.014550399780273438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,255,0.014427199959754944
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,511,0.012630400061607362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,511,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,1023,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,1023,0.012571200728416443
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,float16,2047,0.012495999783277511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,2,128,1,float16,fp8,2047,0.012577599287033081
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,1,0.01234079971909523
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,1,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,3,0.012352000176906585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,3,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,7,0.01247519999742508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,7,0.010630399733781815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,15,0.01239520013332367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,15,0.012412799894809723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,31,0.01244639977812767
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,63,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,31,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,127,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,63,0.011416000127792359
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,127,0.012377600371837615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,255,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,511,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,255,0.01055999994277954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,511,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,1023,0.012415999919176102
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,1023,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,float16,2047,0.012513600289821625
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,1,0.010294400155544281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,4,128,1,float16,fp8,2047,0.012539200484752655
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,1,0.010390400141477584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,3,0.010384000092744827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,3,0.010360000282526016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,7,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,7,0.01035040020942688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,15,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,31,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,15,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,31,0.010337600111961364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,63,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,63,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,127,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,255,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,127,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,255,0.010527999699115753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,511,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,511,0.009520000219345093
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,1023,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,1023,0.01032480001449585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,float16,2047,0.011156799644231797
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,1,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,96,8,128,1,float16,fp8,2047,0.010579200088977813
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,1,0.010500799864530563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,3,0.012540799379348756
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,3,0.010784000158309937
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,7,0.012427199631929398
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,7,0.01061440035700798
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,15,0.012639999389648438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,15,0.011488000303506852
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,31,0.012352000176906585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,31,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,63,0.012403199821710587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,63,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,127,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,127,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,255,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,255,0.010529600083827972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,511,0.014403200149536133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,1023,0.016548800468444824
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,float16,1023,0.016872000694274903
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,1,128,1,float16,fp8,511,0.012455999851226807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,1,0.012606400251388549
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,3,0.012507200241088867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,1,0.012606400251388549
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,3,0.012414400279521943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,7,0.012544000148773193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,15,0.012667199969291687
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,7,0.012675200402736665
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,15,0.012383999675512314
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,31,0.012585599720478059
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,63,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,31,0.012601600587368011
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,63,0.012414400279521943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,127,0.01250240057706833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,255,0.012596799433231354
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,127,0.012636800110340119
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,255,0.011640000343322753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,1023,0.02996160089969635
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,float16,511,0.01661919951438904
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,511,0.014688000082969666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,2,128,1,float16,fp8,1023,0.018622399866580965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,1,0.01666560024023056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,1,0.018716800212860107
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,3,0.0186271995306015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,3,0.016596800088882445
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,7,0.018768000602722167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,7,0.016808000206947327
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,15,0.01860480010509491
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,15,0.017745600640773775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,31,0.018718400597572328
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,31,0.01658080071210861
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,63,0.016579200327396394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,63,0.018756799399852753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,127,0.01863040030002594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,255,0.018803200125694274
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,127,0.01658879965543747
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,511,0.031508800387382505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,255,0.01661120057106018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,511,0.0224031999707222
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,1,0.02274879962205887
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,fp8,1023,0.033081600069999696
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,4,128,1,float16,float16,1023,0.050348800420761106
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,3,0.022703999280929567
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,3,0.018049600720405578
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,1,0.016697600483894348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,7,0.02269600033760071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,15,0.02269600033760071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,7,0.017956799268722533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,15,0.018619200587272643
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,31,0.022668799757957457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,31,0.01802079975605011
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,63,0.02268799990415573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,63,0.01666239947080612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,127,0.022782400250434875
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,127,0.018607999384403228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,255,0.028812798857688903
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,255,0.01880960017442703
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,511,0.047228801250457766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,511,0.029304000735282897
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,float16,1023,0.0821071982383728
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,96,8,128,1,float16,fp8,1023,0.049039998650550844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,1,0.016595199704170227
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,1,0.014739200472831726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,3,0.014582400023937226
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,3,0.01655520051717758
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,7,0.016622400283813475
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,15,0.01658719927072525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,7,0.014505599439144135
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,31,0.016524800658226015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,31,0.014526399970054626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,15,0.014620800316333771
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,63,0.016571199893951415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,63,0.014524799585342408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,127,0.016524800658226015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,127,0.014534400403499603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,float16,255,0.014758400619029999
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,1,128,1,float16,fp8,255,0.014440000057220459
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,1,0.018199999630451203
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,1,0.01589599996805191
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,3,0.018089599907398224
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,3,0.016553600132465363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,7,0.01812800019979477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,7,0.016550399363040924
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,15,0.01759520024061203
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,15,0.0164560005068779
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,31,0.016476799547672272
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,31,0.018006399273872375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,63,0.018534399569034576
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,63,0.016513599455356597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,127,0.018507200479507446
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,127,0.016553600132465363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,float16,255,0.018611200153827667
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,2,128,1,float16,fp8,255,0.01459839940071106
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,1,0.030771198868751525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,1,0.026812800765037538
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,3,0.029758399724960326
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,3,0.026867198944091796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,7,0.029481598734855653
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,7,0.026859200000762938
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,15,0.029728001356124877
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,15,0.026919999718666078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,31,0.029419198632240295
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,31,0.02680320143699646
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,63,0.03107360005378723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,63,0.026790401339530943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,127,0.029207998514175416
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,float16,255,0.03311040103435516
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,255,0.027011200785636902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,4,128,1,float16,fp8,127,0.02712959945201874
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,1,0.037088000774383546
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,1,0.02677760124206543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,3,0.037329599261283875
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,3,0.02688480019569397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,7,0.03705599904060364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,15,0.03696320056915283
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,15,0.026840001344680786
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,7,0.027198401093482972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,31,0.0371535986661911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,31,0.026924800872802735
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,63,0.03713920116424561
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,63,0.02687999904155731
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,127,0.03904159963130951
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,127,0.026955199241638184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,float16,255,0.04773919880390167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,1,0.023046399652957916
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,96,8,128,1,float16,fp8,255,0.03173120021820068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,1,0.02093600034713745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,3,0.022968000173568724
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,3,0.02067359983921051
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,7,0.02301599979400635
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,7,0.020750400424003602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,15,0.022759999334812164
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,15,0.02088640034198761
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,31,0.02414720058441162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,31,0.020657600462436677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,63,0.022843199968338012
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,63,0.020812800526618956
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,float16,127,0.022782400250434875
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,1,128,1,float16,fp8,127,0.020729599893093108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,1,0.02274399995803833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,1,0.026732799410820008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,3,0.025643199682235718
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,3,0.02268960028886795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,7,0.02672159969806671
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,7,0.02261119931936264
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,15,0.025328001379966734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,15,0.022694399952888487
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,31,0.025547200441360475
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,31,0.022644799947738648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,63,0.02645600140094757
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,63,0.022815999388694764
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,float16,127,0.02688319981098175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,2,128,1,float16,fp8,127,0.02263360023498535
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,1,0.05141119956970215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,1,0.046609601378440856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,3,0.05145440101623535
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,3,0.04625279903411865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,7,0.05146399736404419
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,15,0.05145919919013977
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,7,0.04589439928531647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,15,0.04524160027503967
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,31,0.05141440033912659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,31,0.04562079906463623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,63,0.05355839729309082
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,63,0.04551999866962433
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,float16,127,0.05666239857673645
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,4,128,1,float16,fp8,127,0.04695520102977753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,1,0.06575999855995178
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,1,0.045372799038887024
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,3,0.06571840047836304
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,3,0.04535839855670929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,7,0.06576480269432068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,15,0.0658896028995514
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,7,0.04527519941329956
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,15,0.04536480009555817
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,31,0.04524959921836853
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,31,0.06753600239753724
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,63,0.04532960057258606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,63,0.0676576018333435
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,fp8,127,0.04733439981937408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,96,8,128,1,float16,float16,127,0.06799839735031128
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,1,0.018636800348758698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,1,0.01868640035390854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,3,0.018716800212860107
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,3,0.01879200041294098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,7,0.018716800212860107
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,7,0.018643200397491455
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,15,0.018779200315475465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,31,0.01866080015897751
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,15,0.018750399351119995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,31,0.01876640021800995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,63,0.018569600582122803
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,127,0.018750399351119995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,63,0.018760000169277192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,127,0.018593600392341612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,255,0.018772800266742707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,511,0.01658560037612915
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,255,0.018489600718021394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,511,0.016510400176048278
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,1023,0.014526399970054626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,1023,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,float16,2047,0.01483519971370697
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,1,128,1,float16,fp8,2047,0.014558400213718414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,1,0.014535999298095703
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,1,0.014455999433994293
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,3,0.01449279934167862
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,3,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,7,0.014480000734329224
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,7,0.014521600306034088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,15,0.014556799829006196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,15,0.014571200311183929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,31,0.014467200636863709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,31,0.014511999487876893
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,63,0.014526399970054626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,127,0.014523200690746307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,63,0.014496000111103058
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,127,0.014473600685596466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,255,0.014460800588130951
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,255,0.014534400403499603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,511,0.012587200105190276
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,1023,0.012513600289821625
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,511,0.012470400333404541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,1023,0.012438400089740754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,float16,2047,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,1,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,2,128,1,float16,fp8,2047,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,1,0.01056319996714592
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,3,0.010664000362157821
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,3,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,7,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,15,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,7,0.010582400113344192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,15,0.010515200346708298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,31,0.010527999699115753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,31,0.010665600001811982
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,63,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,63,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,127,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,127,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,255,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,511,0.010345599800348281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,255,0.01053600013256073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,511,0.01053600013256073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,1023,0.010619200021028518
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,1023,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,float16,2047,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,1,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,4,128,1,float16,fp8,2047,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,1,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,3,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,3,0.010302399843931198
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,7,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,15,0.01061440035700798
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,7,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,15,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,31,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,63,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,31,0.010923200100660325
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,63,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,127,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,127,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,255,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,255,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,511,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,511,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,1023,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,1023,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,float16,2047,0.013724799454212188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,1,0.03702400028705597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,96,8,128,1,float16,fp8,2047,0.012510399520397186
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,1,0.031062400341033934
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,3,0.03704800009727478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,3,0.0315200001001358
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,7,0.03701440095901489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,7,0.030985599756240843
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,15,0.03705280125141144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,15,0.030980798602104186
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,31,0.03708159923553467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,31,0.031001600623130798
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,float16,63,0.037143999338150026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,1,128,1,float16,fp8,63,0.03097440004348755
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,1,0.043156799674034116
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,1,0.037115201354026794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,3,0.04326080083847046
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,3,0.03713920116424561
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,7,0.04320639967918396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,7,0.03717760145664215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,15,0.0432559996843338
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,15,0.03714720010757446
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,31,0.043252798914909366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,31,0.037110400199890134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,fp8,63,0.037195199728012086
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,2,128,1,float16,float16,63,0.0454479992389679
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,1,0.09642400145530701
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,1,0.08424000144004821
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,3,0.09650880098342896
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,3,0.08422560095787049
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,7,0.09652799963951111
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,7,0.08426880240440368
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,15,0.09711520075798034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,15,0.08418400287628174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,31,0.09864959716796876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,31,0.0845359981060028
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,fp8,63,0.08615999817848205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,4,128,1,float16,float16,63,0.10125759840011597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,1,0.1251871943473816
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,1,0.08219360113143921
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,3,0.08225759863853455
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,3,0.1250399947166443
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,7,0.1251855969429016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,7,0.0822767972946167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,15,0.1259775996208191
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,15,0.08214399814605713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,31,0.1253327965736389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,float16,63,0.12527680397033691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,31,0.08447679877281189
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,96,8,128,1,float16,fp8,63,0.08418239951133728
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,1,0.0630240023136139
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,3,0.06380320191383362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,1,0.053921598196029666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,3,0.05366399884223938
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,7,0.0638592004776001
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,7,0.054201602935791016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,15,0.06406720280647278
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,15,0.053604799509048465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,fp8,31,0.05361440181732178
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,1,128,1,float16,float16,31,0.06593440175056457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,1,0.07762240171432495
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,1,0.0657423973083496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,3,0.06582239866256714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,3,0.07805439829826355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,7,0.07788320183753968
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,7,0.06584159731864929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,15,0.06578080058097839
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,15,0.07961440086364746
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,float16,31,0.07993119955062866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,2,128,1,float16,fp8,31,0.0660207986831665
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,1,0.18659520149230957
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,1,0.16205600500106812
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,3,0.1852288007736206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,3,0.16123679876327515
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,7,0.16028000116348268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,7,0.18706079721450805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,15,0.1610592007637024
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,31,0.18874080181121827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,float16,15,0.1878607988357544
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,4,128,1,float16,fp8,31,0.16083199977874757
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,1,0.1567952036857605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,1,0.24210081100463868
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,3,0.2405535936355591
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,3,0.1579103946685791
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,7,0.24195199012756347
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,7,0.157423996925354
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,15,0.2421056032180786
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,15,0.15813920497894288
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,float16,31,0.24077279567718507
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,1,0.0187376007437706
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,1,0.018697600066661834
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,96,8,128,1,float16,fp8,31,0.15792800188064576
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,7,0.018651199340820313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,3,0.018702399730682374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,3,0.018764799833297728
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,7,0.018590399622917177
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,15,0.018694399297237395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,15,0.018681600689888
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,31,0.01876640021800995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,31,0.018590399622917177
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,63,0.018756799399852753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,63,0.018500800430774688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,127,0.018606400489807128
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,127,0.018612800538539885
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,255,0.018592000007629395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,255,0.018611200153827667
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,511,0.016624000668525696
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,511,0.01656160056591034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,1023,0.014537599682807923
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,float16,2047,0.014851200580596923
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,1023,0.014459200203418732
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,1,128,1,float16,fp8,2047,0.014542399346828461
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,1,0.014542399346828461
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,1,0.014443199336528777
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,3,0.014526399970054626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,3,0.014532800018787383
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,7,0.014588800072669984
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,7,0.014537599682807923
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,15,0.014535999298095703
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,15,0.014521600306034088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,31,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,31,0.014569599926471711
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,63,0.014678399264812469
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,63,0.014478400349617004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,127,0.014534400403499603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,127,0.01467680037021637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,255,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,511,0.013529600203037262
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,511,0.012491200119256973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,255,0.014545600116252898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,1023,0.012515200674533844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,2047,0.014521600306034088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,float16,1023,0.013785600662231445
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,1,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,1,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,2,128,1,float16,fp8,2047,0.01449279934167862
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,3,0.01250080019235611
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,7,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,3,0.012430399656295776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,7,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,15,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,15,0.010582400113344192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,31,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,31,0.010507199913263321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,63,0.012121599912643433
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,63,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,127,0.010923200100660325
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,127,0.010856000334024429
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,255,0.01051200032234192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,255,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,511,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,511,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,1023,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,1023,0.012457600235939026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,float16,2047,0.014707200229167938
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,4,128,1,float16,fp8,2047,0.014542399346828461
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,1,0.010315199941396713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,3,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,1,0.008475200086832047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,3,0.008395200222730636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,7,0.009996800124645234
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,7,0.008472000062465668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,15,0.008399999886751174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,15,0.008593600243330002
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,31,0.010332799702882766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,31,0.008372800052165985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,63,0.01032159999012947
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,63,0.00870719999074936
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,127,0.008371199667453765
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,127,0.00841120034456253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,255,0.008460800349712371
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,511,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,255,0.010390400141477584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,511,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,1023,0.01250240057706833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,1023,0.012382400035858155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,float16,2047,0.027107200026512145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,96,8,128,1,float16,fp8,2047,0.014659200608730317
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,1,0.01655679941177368
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,3,0.01672160029411316
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,3,0.016599999368190767
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,1,0.016545599699020384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,7,0.01672479957342148
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,7,0.01655520051717758
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,15,0.016579200327396394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,31,0.016569599509239197
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,31,0.01661919951438904
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,15,0.016489599645137788
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,63,0.01658080071210861
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,63,0.016527999937534333
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,127,0.01672320067882538
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,127,0.016575999557971954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,255,0.016649599373340606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,255,0.016521599888801575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,511,0.014539200067520141
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,511,0.014560000598430633
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,1023,0.01656160056591034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,1023,0.014574399590492249
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,2047,0.018622399866580965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,2047,0.017073599994182585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,float16,4095,0.03301919996738434
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,1,128,1,float16,fp8,4095,0.022487999498844148
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,1,0.010532800108194351
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,1,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,3,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,3,0.010620799660682679
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,7,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,7,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,15,0.01050880029797554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,15,0.010516799986362457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,31,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,31,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,63,0.01056160032749176
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,63,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,127,0.010550399869680404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,127,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,255,0.010527999699115753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,255,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,511,0.012408000230789185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,511,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,1023,0.014532800018787383
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,1023,0.012520000338554382
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,2047,0.02898080050945282
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,2047,0.016769599914550782
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,float16,4095,0.046856001019477844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,2,128,1,float16,fp8,4095,0.029860800504684447
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,1,0.011812800168991089
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,1,0.010489600151777268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,3,0.012179200351238251
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,3,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,7,0.012375999987125397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,7,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,15,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,15,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,31,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,31,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,63,0.012455999851226807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,63,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,127,0.012433599680662155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,127,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,255,0.012430399656295776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,255,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,511,0.014556799829006196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,511,0.012580800056457519
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,1023,0.0165120005607605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,2047,0.04632799923419952
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,1023,0.02902719974517822
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,2047,0.029691201448440552
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,float16,4095,0.080622398853302
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,1,0.012591999769210816
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,4,128,1,float16,fp8,4095,0.04811680018901825
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,3,0.012630400061607362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,1,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,7,0.01266240030527115
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,3,0.012612800300121307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,7,0.012401600182056428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,15,0.012559999525547028
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,31,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,15,0.012606400251388549
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,63,0.012547199428081513
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,31,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,63,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,127,0.012495999783277511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,127,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,255,0.012622399628162384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,255,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,511,0.0288239985704422
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,511,0.014852799475193024
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,1023,0.04685760140419006
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,1023,0.029891198873519896
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,2047,0.08239359855651855
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,2047,0.04795680046081543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,1,0.014483200013637542
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,float16,4095,0.15345760583877563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,64,8,128,1,float16,fp8,4095,0.0853872001171112
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,1,0.01459839940071106
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,7,0.014524799585342408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,3,0.014611199498176575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,3,0.014688000082969666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,7,0.014484800398349762
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,15,0.014739200472831726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,15,0.01465120017528534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,31,0.014521600306034088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,31,0.014472000300884247
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,63,0.014454400539398194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,63,0.014614400267601014
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,127,0.01462399959564209
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,255,0.014688000082969666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,127,0.014511999487876893
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,255,0.01446399986743927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,511,0.01438560038805008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,511,0.012412799894809723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,1023,0.012334399670362473
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,1023,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,2047,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,2047,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,float16,4095,0.014035199582576752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,1,128,1,float16,fp8,4095,0.012515200674533844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,1,0.012403199821710587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,1,0.012563200294971466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,3,0.012382400035858155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,3,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,7,0.012382400035858155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,7,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,15,0.012436799705028534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,15,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,31,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,31,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,63,0.012379200011491776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,63,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,127,0.012399999797344208
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,127,0.012371200323104858
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,255,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,255,0.012849600613117218
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,511,0.012404800206422806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,511,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,1023,0.010526400059461594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,1023,0.010529600083827972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,2047,0.012436799705028534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,2047,0.011513599753379821
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,float16,4095,0.012611199915409089
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,2,128,1,float16,fp8,4095,0.012503999471664428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,1,0.010513599961996078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,1,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,3,0.010527999699115753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,7,0.010579200088977813
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,3,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,7,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,15,0.010471999645233154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,15,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,31,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,31,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,63,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,63,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,127,0.010567999631166457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,255,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,127,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,255,0.010361599922180175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,511,0.010636799782514573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,511,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,1023,0.01053439974784851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,1023,0.010526400059461594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,2047,0.010782399773597717
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,2047,0.010859200358390808
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,float16,4095,0.012503999471664428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,4,128,1,float16,fp8,4095,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,1,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,1,0.008686400204896926
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,3,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,3,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,7,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,7,0.008404800295829773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,15,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,15,0.00843520015478134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,31,0.009369599819183349
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,63,0.010344000160694122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,31,0.01035040020942688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,63,0.00843520015478134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,127,0.010331200063228607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,127,0.01032159999012947
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,255,0.00928800031542778
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,511,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,255,0.009244800359010697
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,511,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,1023,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,1023,0.01035040020942688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,2047,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,2047,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,float16,4095,0.012531200051307678
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,64,8,128,1,float16,fp8,4095,0.012515200674533844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,1,0.014664000272750855
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,3,0.014587199687957764
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,1,0.014705599844455719
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,3,0.014699199795722961
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,7,0.014575999975204468
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,15,0.014641599357128143
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,7,0.014620800316333771
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,31,0.014696000516414643
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,15,0.01451680064201355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,31,0.014713600277900696
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,63,0.014655999839305878
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,127,0.014654399454593658
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,63,0.01454399973154068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,255,0.014582400023937226
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,127,0.014703999459743499
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,255,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,511,0.014419199526309967
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,511,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,1023,0.01244639977812767
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,1023,0.012415999919176102
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,2047,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,2047,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,float16,4095,0.014427199959754944
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,1,128,1,float16,fp8,4095,0.01266240030527115
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,1,0.011409600079059602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,1,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,3,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,3,0.010489600151777268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,7,0.01252799928188324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,7,0.012451200187206269
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,15,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,15,0.012387199699878693
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,31,0.012415999919176102
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,31,0.012385600060224534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,63,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,63,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,127,0.012443199753761292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,127,0.012399999797344208
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,255,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,255,0.011646399646997452
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,511,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,511,0.01053600013256073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,1023,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,1023,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,2047,0.012556800246238708
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,float16,4095,0.01268640011548996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,2047,0.012455999851226807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,1,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,2,128,1,float16,fp8,4095,0.012654399871826172
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,3,0.010515200346708298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,1,0.010480000078678131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,3,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,7,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,7,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,15,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,15,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,31,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,31,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,63,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,63,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,127,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,127,0.010523200035095215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,255,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,255,0.010521599650382995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,511,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,511,0.010513599961996078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,1023,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,1023,0.01056319996714592
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,2047,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,2047,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,float16,4095,0.012751999497413635
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,4,128,1,float16,fp8,4095,0.012681600451469422
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,1,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,1,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,3,0.009848000109195709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,3,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,7,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,7,0.009105599671602248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,15,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,15,0.00926240012049675
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,31,0.010241600126028061
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,31,0.009731200337409974
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,63,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,63,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,127,0.010328000038862228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,127,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,255,0.00841279998421669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,255,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,511,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,511,0.008904000371694564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,1023,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,1023,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,2047,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,2047,0.010552000254392624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,float16,4095,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,64,8,128,1,float16,fp8,4095,0.01247519999742508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,1,0.016521599888801575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,1,0.016516800224781036
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,3,0.016569599509239197
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,3,0.016489599645137788
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,7,0.016633599996566772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,7,0.016256000101566314
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,15,0.016441600024700166
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,15,0.016524800658226015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,31,0.016680000722408293
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,31,0.016415999829769136
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,63,0.01653759926557541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,63,0.01615840047597885
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,127,0.016436800360679626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,127,0.015139199793338776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,255,0.016595199704170227
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,255,0.015248000621795654
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,511,0.014537599682807923
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,1023,0.012572799623012543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,1023,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,511,0.014563199877738953
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,2047,0.014715200662612915
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,float16,4095,0.01720159947872162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,4095,0.01658399999141693
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,1,128,1,float16,fp8,2047,0.014532800018787383
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,1,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,1,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,3,0.01239679977297783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,3,0.011153600364923476
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,7,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,7,0.011406400054693223
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,15,0.012256000190973282
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,15,0.012392000108957291
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,31,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,31,0.01218079999089241
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,63,0.01242239996790886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,63,0.011847999691963196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,127,0.012489599734544754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,255,0.012449599802494049
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,255,0.010768000036478043
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,127,0.012388800084590913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,511,0.010468800365924836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,511,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,1023,0.012516799569129943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,1023,0.01252640038728714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,2047,0.014798399806022645
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,2047,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,float16,4095,0.030142399668693542
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,1,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,2,128,1,float16,fp8,4095,0.01860159933567047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,1,0.008523199707269669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,3,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,3,0.008551999926567078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,7,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,7,0.008515200018882752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,15,0.01035040020942688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,15,0.008451200276613235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,31,0.008555199950933456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,63,0.008367999643087386
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,31,0.008529599756002426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,127,0.010342399775981902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,63,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,127,0.008921600133180618
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,255,0.008904000371694564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,255,0.008689600229263305
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,511,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,511,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,1023,0.012542399764060973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,1023,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,2047,0.02724800109863281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,2047,0.014662399888038635
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,float16,4095,0.045291200280189514
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,1,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,4,128,1,float16,fp8,4095,0.02813279926776886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,1,0.01061440035700798
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,3,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,3,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,7,0.009625600278377533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,7,0.010728000104427338
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,15,0.01053600013256073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,15,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,31,0.010579200088977813
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,31,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,63,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,127,0.010571199655532836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,63,0.010518400371074677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,127,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,255,0.010532800108194351
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,255,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,511,0.012567999958992004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,511,0.010603199899196624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,1023,0.027076798677444457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,1023,0.01451359987258911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,2047,0.045212799310684205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,2047,0.028703999519348145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,float16,4095,0.08000320196151733
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,1,0.014569599926471711
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,64,8,128,1,float16,fp8,4095,0.04737919867038727
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,1,0.014684799313545226
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,3,0.014564800262451171
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,3,0.01467680037021637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,7,0.015211200714111328
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,15,0.016204799711704253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,7,0.014612799882888794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,15,0.014580799639225006
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,31,0.014737600088119506
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,63,0.0164560005068779
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,31,0.014584000408649444
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,63,0.014683200418949128
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,127,0.014571200311183929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,127,0.0165120005607605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,255,0.014689600467681885
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,255,0.014556799829006196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,511,0.014452800154685974
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,511,0.012641599774360657
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,1023,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,1023,0.012387199699878693
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,2047,0.012507200241088867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,float16,4095,0.014470399916172027
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,2047,0.012414400279521943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,1,128,1,float16,fp8,4095,0.012606400251388549
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,1,0.012491200119256973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,1,0.012289600074291229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,3,0.012505599856376648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,3,0.012411200255155564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,7,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,7,0.012595200538635254
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,15,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,15,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,31,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,31,0.012462399899959564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,63,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,63,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,127,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,127,0.01244639977812767
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,255,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,255,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,511,0.0112527996301651
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,511,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,1023,0.010632000118494033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,1023,0.010564800351858139
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,2047,0.012409599870443344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,2047,0.01250240057706833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,float16,4095,0.014521600306034088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,2,128,1,float16,fp8,4095,0.012697599828243256
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,1,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,1,0.010507199913263321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,3,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,3,0.01051200032234192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,7,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,7,0.010699199885129929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,15,0.01053759977221489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,15,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,31,0.010571199655532836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,31,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,63,0.01053439974784851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,63,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,127,0.010507199913263321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,127,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,255,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,255,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,511,0.010545600205659866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,511,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,1023,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,1023,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,2047,0.01250240057706833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,2047,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,float16,4095,0.01420000046491623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,4,128,1,float16,fp8,4095,0.012491200119256973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,1,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,1,0.00873439982533455
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,3,0.008427199721336365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,3,0.008537600189447403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,7,0.008399999886751174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,7,0.00856959968805313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,15,0.009665600210428237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,15,0.00841120034456253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,31,0.008495999872684479
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,31,0.008683200180530547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,63,0.008392000198364257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,63,0.008551999926567078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,127,0.008614400029182434
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,127,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,255,0.009139200299978256
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,511,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,255,0.008764799684286118
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,511,0.008535999804735184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,1023,0.009574399888515472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,2047,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,1023,0.009017600119113922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,float16,4095,0.012638400495052337
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,2047,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,64,8,128,1,float16,fp8,4095,0.010684800148010255
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,1,0.010480000078678131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,1,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,3,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,3,0.010334400087594986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,7,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,15,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,7,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,15,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,31,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,31,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,63,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,127,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,63,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,127,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,255,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,511,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,255,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,511,0.010558400303125381
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,1023,0.012600000202655792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,1023,0.014468799531459808
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,float16,2047,0.029025599360466003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,1,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,1,0.012547199428081513
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,1,128,1,float16,fp8,2047,0.01820800006389618
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,3,0.012513600289821625
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,3,0.012702399492263794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,7,0.013460800051689148
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,7,0.01252799928188324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,15,0.014414399862289429
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,15,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,31,0.012516799569129943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,31,0.012692800164222718
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,63,0.013801600039005279
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,127,0.014444799721240997
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,63,0.012455999851226807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,127,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,255,0.013817599415779114
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,255,0.012628799676895142
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,511,0.016540800034999848
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,511,0.014713600277900696
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,1023,0.02975519895553589
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,1023,0.01863040030002594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,float16,2047,0.04779520034790039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,1,0.014561599493026734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,1,0.01257600039243698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,3,0.014596800506114959
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,2,128,1,float16,fp8,2047,0.03287999927997589
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,3,0.01371999979019165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,7,0.01451359987258911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,7,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,15,0.014732800424098969
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,15,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,31,0.014670400321483612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,31,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,63,0.014748799800872802
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,63,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,127,0.014660799503326416
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,127,0.013195200264453888
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,255,0.015345600247383118
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,511,0.028996801376342772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,255,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,511,0.01669600009918213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,1023,0.04805760085582733
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,1023,0.030982398986816408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,float16,2047,0.08373919725418091
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,4,128,1,float16,fp8,2047,0.04953919947147369
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,1,0.01669279932975769
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,1,0.01462240070104599
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,3,0.01658399999141693
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,3,0.014812800288200378
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,7,0.016631999611854555
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,7,0.014657600224018097
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,15,0.016604800522327424
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,15,0.014727999269962311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,31,0.016579200327396394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,31,0.014979200065135955
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,63,0.016603200137615202
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,127,0.017558400332927705
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,63,0.016739200055599212
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,127,0.014552000164985656
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,255,0.028907200694084166
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,255,0.016516800224781036
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,511,0.04538240134716034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,511,0.029084798693656922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,1023,0.0478767991065979
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,1023,0.08127520084381104
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,1,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,float16,2047,0.15106719732284546
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,64,8,128,1,float16,fp8,2047,0.08437439799308777
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,1,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,3,0.013646399974822998
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,7,0.012664000689983367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,3,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,7,0.01255359947681427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,15,0.012630400061607362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,15,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,31,0.013971200585365296
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,31,0.012375999987125397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,63,0.012520000338554382
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,63,0.012503999471664428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,127,0.012545600533485413
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,127,0.012507200241088867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,255,0.012647999823093415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,255,0.01236959993839264
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,fp8,511,0.014457599818706512
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,1,128,1,float16,float16,511,0.016590400040149687
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,1,0.018670399487018586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,1,0.017878399789333345
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,3,0.0186271995306015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,3,0.01857919991016388
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,7,0.01889439970254898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,7,0.017108799517154695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,15,0.019310399889945984
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,15,0.01783200055360794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,31,0.01868959963321686
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,31,0.01863040030002594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,63,0.019739200174808503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,63,0.017100800573825837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,127,0.018878400325775146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,127,0.018588800728321076
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,255,0.01897439956665039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,255,0.018639999628067016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,float16,511,0.03295199871063233
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,2,128,1,float16,fp8,511,0.02274080067873001
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,1,0.022777600586414336
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,1,0.018566399812698364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,3,0.02268960028886795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,7,0.022614400088787078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,3,0.018540799617767334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,7,0.018643200397491455
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,15,0.02266719937324524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,15,0.01849440038204193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,31,0.02263839989900589
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,31,0.018515199422836304
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,63,0.02268799990415573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,63,0.018628799915313722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,127,0.022628800570964815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,127,0.01852799952030182
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,255,0.028942400217056276
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,255,0.01852000057697296
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,fp8,511,0.03054080009460449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,4,128,1,float16,float16,511,0.04736000001430511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,1,0.026807999610900878
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,1,0.022750400006771088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,3,0.026788800954818726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,3,0.02290399968624115
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,7,0.0268095999956131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,7,0.02287199944257736
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,15,0.026926401257514953
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,31,0.02688480019569397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,31,0.022865599393844603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,15,0.02274399995803833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,63,0.026924800872802735
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,63,0.022756800055503845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,127,0.030964800715446474
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,127,0.023027199506759643
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,255,0.04738239943981171
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,255,0.030350399017333985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,float16,511,0.0822704017162323
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,64,8,128,1,float16,fp8,511,0.04757600128650665
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,1,0.018580800294876097
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,1,0.016510400176048278
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,3,0.018507200479507446
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,3,0.016595199704170227
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,7,0.01849119961261749
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,7,0.016606399416923524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,15,0.018087999522686006
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,15,0.01663520038127899
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,31,0.018331199884414673
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,31,0.01658399999141693
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,63,0.018335999548435213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,63,0.01653279960155487
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,127,0.016568000614643096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,127,0.018615999817848207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,1,0.03089439868927002
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,float16,255,0.0186271995306015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,1,128,1,float16,fp8,255,0.016551999747753142
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,3,0.03089280128479004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,1,0.027062401175498962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,3,0.027419200539588927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,7,0.030854400992393494
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,7,0.026923200488090514
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,15,0.03088639974594116
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,15,0.028360000252723692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,31,0.030665600299835206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,31,0.027137601375579835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,63,0.030895999073982237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,63,0.02808319926261902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,127,0.030934399366378783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,127,0.028566399216651918
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,fp8,255,0.02876800000667572
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,2,128,1,float16,float16,255,0.03510560095310211
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,1,0.03711360096931458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,1,0.026995199918746948
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,3,0.03707199990749359
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,3,0.026796799898147584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,7,0.036977601051330564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,7,0.026851201057434083
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,15,0.026952001452445983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,15,0.03704479932785034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,31,0.03705599904060364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,31,0.027009600400924684
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,63,0.03709119856357575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,63,0.026819199323654175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,127,0.0390751987695694
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,127,0.027031999826431275
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,float16,255,0.048614400625228885
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,4,128,1,float16,fp8,255,0.03131200075149536
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,1,0.045300799608230594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,1,0.039094400405883786
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,3,0.045310398936271666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,3,0.03906719982624054
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,7,0.04533120095729828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,7,0.03902879953384399
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,15,0.04525760114192963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,15,0.03911199867725372
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,31,0.04725759923458099
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,31,0.03912320137023926
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,63,0.04769439995288849
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,63,0.039243200421333314
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,127,0.051419198513031006
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,127,0.041249600052833554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,float16,255,0.08424479961395263
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,64,8,128,1,float16,fp8,255,0.05127840042114258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,1,0.014496000111103058
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,3,0.014537599682807923
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,3,0.014524799585342408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,1,0.014732800424098969
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,7,0.014539200067520141
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,7,0.014542399346828461
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,15,0.01643519997596741
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,15,0.014601600170135499
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,31,0.014611199498176575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,31,0.015352000296115876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,63,0.014580799639225006
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,63,0.014524799585342408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,127,0.014684799313545226
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,127,0.014521600306034088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,255,0.01462399959564209
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,255,0.014593599736690522
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,511,0.014505599439144135
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,1023,0.012652799487113953
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,511,0.014084799587726593
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,1023,0.012529599666595458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,2047,0.01260959953069687
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,2047,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,float16,4095,0.014504000544548035
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,1,128,1,float16,fp8,4095,0.01372160017490387
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,1,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,1,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,3,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,3,0.012464000284671784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,7,0.012352000176906585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,7,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,15,0.012436799705028534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,15,0.012564800679683685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,31,0.01250080019235611
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,31,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,63,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,63,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,127,0.012588800489902496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,255,0.012495999783277511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,127,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,255,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,511,0.012513600289821625
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,511,0.012452799826860428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,1023,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,1023,0.012428800016641617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,2047,0.012470400333404541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,2047,0.012676799297332763
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,float16,4095,0.01451359987258911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,2,128,1,float16,fp8,4095,0.014508800208568573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,1,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,1,0.01061599999666214
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,3,0.010339199751615524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,3,0.01034879982471466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,7,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,7,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,15,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,15,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,31,0.010585600137710571
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,63,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,31,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,127,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,127,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,63,0.010571199655532836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,255,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,255,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,511,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,511,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,1023,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,1023,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,2047,0.010711999982595444
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,2047,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,fp8,4095,0.012443199753761292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,4,128,1,float16,float16,4095,0.012561599910259246
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,1,0.00841120034456253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,1,0.008483199775218964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,3,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,3,0.008897600322961807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,7,0.01003360003232956
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,7,0.008582399785518646
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,15,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,15,0.00846880003809929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,31,0.010175999999046326
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,31,0.009513600170612336
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,63,0.010129600018262862
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,63,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,127,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,127,0.008387199789285659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,255,0.010238400101661682
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,255,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,511,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,1023,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,511,0.009486400336027146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,1023,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,2047,0.012548799812793731
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,2047,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,float16,4095,0.027830401062965394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,64,8,128,1,float16,fp8,4095,0.014497600495815277
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,1,0.02677280008792877
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,1,0.02476799935102463
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,3,0.0267984002828598
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,7,0.02690880000591278
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,3,0.02276480048894882
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,15,0.02675360143184662
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,7,0.024539199471473695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,15,0.0227743998169899
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,31,0.026756799221038817
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,31,0.02470400035381317
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,63,0.025543999671936036
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,63,0.024672000110149382
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,float16,127,0.027139198780059815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,1,128,1,float16,fp8,127,0.022835199534893037
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,1,0.05209919810295105
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,1,0.04727199971675873
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,3,0.05140320062637329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,3,0.04742560088634491
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,7,0.05162879824638367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,7,0.047366398572921756
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,15,0.05145599842071533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,15,0.04727360010147095
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,31,0.04736959934234619
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,31,0.051820802688598636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,63,0.05440959930419922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,63,0.04738560020923614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,float16,127,0.05814880132675171
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,2,128,1,float16,fp8,127,0.048347198963165285
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,1,0.06584799885749817
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,1,0.04732640087604523
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,3,0.06566399931907654
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,3,0.04599359929561615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,7,0.0657472014427185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,7,0.04589279890060425
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,15,0.04645279943943024
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,15,0.0657151997089386
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,31,0.06621760129928589
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,31,0.0468639999628067
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,63,0.06726719737052918
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,63,0.04735519886016846
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,float16,127,0.06797919869422912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,4,128,1,float16,fp8,127,0.04879679977893829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,1,0.08267679810523987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,1,0.06992639899253845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,3,0.08225439786911011
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,3,0.0699728012084961
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,7,0.08425599932670594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,7,0.069896000623703
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,15,0.07003520131111145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,15,0.0845408022403717
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,31,0.08442080020904541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,31,0.06992319822311402
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,63,0.08438720107078553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,63,0.07175040245056152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,float16,127,0.09364640116691589
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,64,8,128,1,float16,fp8,127,0.07360320091247559
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,1,0.04329760074615478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,3,0.04334079921245575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,1,0.03917919993400574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,3,0.03920960128307342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,7,0.043249601125717164
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,7,0.03931199908256531
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,15,0.04355520009994507
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,15,0.03924640119075775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,31,0.04547359943389893
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,31,0.03922399878501892
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,float16,63,0.04657599925994873
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,1,128,1,float16,fp8,63,0.03924799859523773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,1,0.09661759734153748
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,1,0.08635680079460144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,3,0.098089599609375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,3,0.08620479702949524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,7,0.09839360117912292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,15,0.0992255985736847
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,7,0.08633440136909484
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,15,0.08617439866065979
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,31,0.10071840286254882
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,31,0.08732479810714722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,float16,63,0.10262399911880493
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,1,0.12318079471588135
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,2,128,1,float16,fp8,63,0.08841440081596375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,1,0.08419520258903504
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,3,0.12307519912719726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,3,0.08391039967536926
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,7,0.12311840057373047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,7,0.0842464029788971
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,15,0.12321439981460572
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,15,0.08429279923439026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,31,0.12407679557800293
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,31,0.08530719876289368
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,float16,63,0.12415839433670044
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,4,128,1,float16,fp8,63,0.08636159896850586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,1,0.15794559717178344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,1,0.1313024044036865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,3,0.1590767979621887
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,3,0.13149919509887695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,7,0.15931040048599243
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,7,0.1314255952835083
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,15,0.15799520015716553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,15,0.13308800458908082
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,31,0.1591439962387085
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,31,0.13307039737701415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,float16,63,0.16019840240478517
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,64,8,128,1,float16,fp8,63,0.1349951982498169
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,1,0.016513599455356597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,1,0.01624000072479248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,3,0.01653279960155487
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,3,0.016518400609493257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,7,0.016575999557971954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,7,0.01603199988603592
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,15,0.016543999314308167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,15,0.015675200521945952
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,31,0.016519999504089354
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,31,0.016467200219631196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,63,0.016579200327396394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,63,0.015358400344848634
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,127,0.016515199840068818
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,255,0.01645440012216568
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,127,0.016047999262809753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,255,0.014644800126552582
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,511,0.014443199336528777
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,511,0.012572799623012543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,1023,0.012601600587368011
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,1023,0.01257600039243698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,2047,0.012518399953842163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,2047,0.012585599720478059
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,float16,4095,0.014788800477981567
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,1,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,1,128,1,float16,fp8,4095,0.01454080045223236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,3,0.010648000240325927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,1,0.010556799918413162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,7,0.010521599650382995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,3,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,7,0.010515200346708298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,15,0.010704000294208527
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,15,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,31,0.011135999858379365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,31,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,63,0.010529600083827972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,63,0.010644800215959548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,127,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,127,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,255,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,255,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,511,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,511,0.010545600205659866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,1023,0.0105103999376297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,1023,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,2047,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,2047,0.012374400347471236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,float16,4095,0.014529600739479065
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,1,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,2,128,1,float16,fp8,4095,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,1,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,3,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,3,0.010523200035095215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,7,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,7,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,15,0.010547199845314026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,15,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,63,0.010471999645233154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,31,0.010566399991512298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,31,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,63,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,127,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,127,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,255,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,255,0.010465600341558457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,511,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,511,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,1023,0.010542400181293488
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,1023,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,2047,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,2047,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,float16,4095,0.028859201073646545
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,4,128,1,float16,fp8,4095,0.014897599816322327
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,1,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,1,0.00918240025639534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,3,0.008441600203514098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,3,0.008422400057315826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,7,0.008548799902200699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,7,0.008398400247097015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,15,0.008404800295829773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,15,0.008423999696969987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,31,0.008452799916267396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,31,0.008388800173997879
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,63,0.00904960036277771
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,63,0.008380799740552902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,127,0.008684799820184708
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,127,0.008448000252246856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,255,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,255,0.008423999696969987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,511,0.009915199875831605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,511,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,1023,0.012491200119256973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,1023,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,2047,0.027100801467895508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,2047,0.012464000284671784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,1,0.01234079971909523
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,float16,4095,0.043945598602294925
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,64,8,128,1,float16,fp8,4095,0.027657601237297057
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,1,0.010814400017261505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,3,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,3,0.012408000230789185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,7,0.011737599968910217
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,7,0.012510399520397186
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,15,0.012532800436019897
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,31,0.0123648002743721
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,31,0.012372799962759019
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,15,0.011923199892044068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,63,0.0123648002743721
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,63,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,127,0.01250080019235611
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,127,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,255,0.012433599680662155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,255,0.010763200372457505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,511,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,511,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,1023,0.012556800246238708
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,1023,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,2047,0.014620800316333771
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,2047,0.01451359987258911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,4095,0.02911520004272461
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,4095,0.018681600689888
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,float16,8191,0.04725599884986877
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,1,0.01029760017991066
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,1,0.008475200086832047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,1,128,1,float16,fp8,8191,0.030822399258613586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,3,0.010326399654150008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,3,0.008417599648237229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,7,0.010152000188827514
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,7,0.009948799759149552
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,15,0.010334400087594986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,15,0.009721600264310837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,31,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,63,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,31,0.010000000149011612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,63,0.008481600135564805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,127,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,127,0.008401600271463394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,255,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,255,0.008488000184297562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,511,0.010564800351858139
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,511,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,1023,0.012561599910259246
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,1023,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,2047,0.02701599895954132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,2047,0.014601600170135499
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,4095,0.04525279998779297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,4095,0.0289792001247406
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,float16,8191,0.07871840000152588
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,2,128,1,float16,fp8,8191,0.045396798849105836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,1,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,3,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,1,0.010332799702882766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,3,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,7,0.010471999645233154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,7,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,15,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,15,0.010310400277376175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,31,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,63,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,31,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,63,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,127,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,127,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,255,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,511,0.012464000284671784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,255,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,511,0.010635200142860412
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,1023,0.027958399057388304
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,1023,0.01451680064201355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,2047,0.04535999894142151
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,2047,0.028401601314544677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,4095,0.07995679974555969
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,4095,0.04639680087566376
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,1,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,float16,8191,0.1492400050163269
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,4,128,1,float16,fp8,8191,0.08235520124435425
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,1,0.012435200065374375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,7,0.012518399953842163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,3,0.012715199589729309
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,3,0.012380799651145935
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,7,0.012505599856376648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,15,0.012577599287033081
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,15,0.012435200065374375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,31,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,31,0.012403199821710587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,63,0.01260959953069687
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,63,0.012457600235939026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,127,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,127,0.012577599287033081
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,255,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,255,0.012444800138473511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,511,0.014659200608730317
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,511,0.0288783997297287
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,1023,0.046744000911712644
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,1023,0.02906239926815033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,2047,0.08200799822807311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,2047,0.04776639938354492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,4095,0.15234559774398804
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,4095,0.08488479852676392
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,1,0.010595200210809707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,float16,8191,0.29465279579162595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,1,0.010564800351858139
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,32,8,128,1,float16,fp8,8191,0.15972000360488892
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,3,0.010543999820947647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,3,0.012462399899959564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,7,0.01053759977221489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,7,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,15,0.01242239996790886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,31,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,15,0.010564800351858139
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,31,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,63,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,63,0.01053439974784851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,127,0.012430399656295776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,255,0.012398400157690049
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,127,0.010526400059461594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,255,0.011049599945545196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,511,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,511,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,1023,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,1023,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,2047,0.012452799826860428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,2047,0.010595200210809707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,4095,0.012695999443531036
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,float16,8191,0.014595200121402741
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,4095,0.012433599680662155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,1,128,1,float16,fp8,8191,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,1,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,1,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,3,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,3,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,7,0.010480000078678131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,7,0.010344000160694122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,15,0.01037440001964569
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,31,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,15,0.010585600137710571
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,31,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,63,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,127,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,63,0.010566399991512298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,127,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,255,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,511,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,255,0.010569600015878677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,511,0.010540799796581268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,1023,0.010507199913263321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,2047,0.012444800138473511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,1023,0.010694400221109391
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,2047,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,4095,0.01255040019750595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,4095,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,float16,8191,0.014555199444293976
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,2,128,1,float16,fp8,8191,0.014475199580192565
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,1,0.00939679965376854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,1,0.008667200058698653
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,3,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,3,0.008380799740552902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,7,0.009470400214195252
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,7,0.010344000160694122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,15,0.009752000123262406
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,15,0.00838399976491928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,31,0.01035040020942688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,63,0.008584000170230865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,31,0.008406399935483932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,63,0.01032480001449585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,127,0.010339199751615524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,127,0.008483199775218964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,255,0.010344000160694122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,255,0.008489599823951722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,511,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,511,0.009775999933481216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,1023,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,2047,0.010515200346708298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,1023,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,2047,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,4095,0.01239359974861145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,4095,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,float16,8191,0.01451359987258911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,4,128,1,float16,fp8,8191,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,1,0.008513599634170532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,1,0.008636800199747085
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,3,0.010302399843931198
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,3,0.0083856001496315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,7,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,7,0.010344000160694122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,15,0.008444800227880477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,15,0.0089199997484684
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,31,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,31,0.008459199965000153
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,63,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,63,0.008462399989366532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,127,0.009307199716567993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,127,0.008380799740552902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,255,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,255,0.008449599891901017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,511,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,1023,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,511,0.010334400087594986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,1023,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,2047,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,2047,0.010513599961996078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,4095,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,float16,8191,0.014449599385261535
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,4095,0.012438400089740754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,32,8,128,1,float16,fp8,8191,0.01255200058221817
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,1,0.012470400333404541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,1,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,3,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,3,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,7,0.012567999958992004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,7,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,15,0.012414400279521943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,15,0.012430399656295776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,31,0.010833600163459777
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,31,0.012470400333404541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,63,0.012521600723266602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,63,0.012275200337171555
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,127,0.01252799928188324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,127,0.012433599680662155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,255,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,255,0.012430399656295776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,511,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,1023,0.012390399724245072
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,511,0.011655999720096588
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,1023,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,2047,0.012427199631929398
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,2047,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,4095,0.012583999335765839
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,4095,0.012464000284671784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,float16,8191,0.014535999298095703
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,1,128,1,float16,fp8,8191,0.014507199823856353
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,1,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,1,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,3,0.010620799660682679
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,3,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,7,0.010555200278759003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,7,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,15,0.010499200224876404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,15,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,31,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,63,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,31,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,127,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,63,0.010555200278759003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,255,0.010636799782514573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,127,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,255,0.01061440035700798
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,511,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,511,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,1023,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,2047,0.01080000028014183
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,1023,0.011052799969911575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,4095,0.01252480000257492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,2047,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,4095,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,float16,8191,0.01462240070104599
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,2,128,1,float16,fp8,8191,0.014526399970054626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,1,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,3,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,1,0.008539199829101562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,3,0.008420799672603608
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,7,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,7,0.008388800173997879
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,15,0.010311999917030334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,15,0.008388800173997879
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,31,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,63,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,31,0.008425600081682205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,127,0.008614400029182434
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,63,0.010326399654150008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,127,0.008507200330495835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,255,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,255,0.008451200276613235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,511,0.010361599922180175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,511,0.009284800291061402
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,1023,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,1023,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,2047,0.010480000078678131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,2047,0.010628800094127654
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,4095,0.012561599910259246
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,4095,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,float16,8191,0.014510400593280792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,1,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,4,128,1,float16,fp8,8191,0.012638400495052337
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,3,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,1,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,3,0.008641599863767623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,7,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,15,0.010291200131177902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,7,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,15,0.00912960022687912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,31,0.010329599678516387
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,31,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,63,0.010384000092744827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,63,0.009419199824333192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,127,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,127,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,255,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,255,0.010345599800348281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,511,0.010499200224876404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,511,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,1023,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,1023,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,2047,0.011315199732780456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,2047,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,4095,0.012470400333404541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,4095,0.012372799962759019
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,float16,8191,0.014696000516414643
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,1,0.0105103999376297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,32,8,128,1,float16,fp8,8191,0.012505599856376648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,3,0.010571199655532836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,3,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,1,0.012377600371837615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,7,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,15,0.01236959993839264
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,7,0.011596799641847611
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,15,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,31,0.012436799705028534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,31,0.010582400113344192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,63,0.012411200255155564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,63,0.011292800307273865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,127,0.012404800206422806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,127,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,255,0.010500799864530563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,255,0.010494399815797806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,511,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,511,0.010555200278759003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,1023,0.010590399801731109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,1023,0.010494399815797806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,2047,0.01249760016798973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,2047,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,4095,0.014772799611091614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,4095,0.01451680064201355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,float16,8191,0.02890079915523529
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,1,128,1,float16,fp8,8191,0.018606400489807128
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,1,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,1,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,3,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,3,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,7,0.01050880029797554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,7,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,15,0.0105103999376297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,15,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,31,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,31,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,63,0.010553599894046783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,127,0.010555200278759003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,63,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,127,0.010585600137710571
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,255,0.010632000118494033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,255,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,511,0.010529600083827972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,1023,0.011019200086593628
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,511,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,1023,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,2047,0.012611199915409089
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,2047,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,4095,0.028760001063346863
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,4095,0.015148800611495972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,float16,8191,0.045368000864982605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,1,0.008406399935483932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,2,128,1,float16,fp8,8191,0.028947201371192933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,1,0.008524800091981888
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,3,0.00836160033941269
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,3,0.008382400125265121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,7,0.008417599648237229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,7,0.008363199979066848
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,15,0.008486399799585343
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,15,0.008472000062465668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,31,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,31,0.00835679993033409
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,63,0.009163200110197067
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,63,0.008564800024032593
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,127,0.008382400125265121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,127,0.008635199815034866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,255,0.008353599905967712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,255,0.008340799808502197
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,511,0.010516799986362457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,511,0.010224000364542008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,1023,0.012427199631929398
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,1023,0.01072160005569458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,2047,0.027182400226593018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,2047,0.012451200187206269
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,4095,0.04506079852581024
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,4095,0.0270687997341156
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,1,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,float16,8191,0.07837759852409362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,4,128,1,float16,fp8,8191,0.04539200067520142
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,1,0.00878399983048439
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,3,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,3,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,7,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,7,0.009312000125646591
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,15,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,31,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,31,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,15,0.009564799815416336
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,63,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,63,0.010360000282526016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,127,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,127,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,255,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,255,0.009700799733400345
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,511,0.012508800625801087
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,511,0.012191999703645706
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,1023,0.02699039876461029
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,1023,0.014472000300884247
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,2047,0.0286080002784729
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,2047,0.04525760114192963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,4095,0.0799888014793396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,4095,0.047307199239730834
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,1,0.012444800138473511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,float16,8191,0.1498960018157959
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,32,8,128,1,float16,fp8,8191,0.08376160264015198
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,1,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,3,0.011460799723863602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,3,0.012401600182056428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,7,0.011822400242090225
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,15,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,7,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,15,0.011534400284290314
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,31,0.012411200255155564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,31,0.010889600217342376
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,63,0.012265600264072418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,63,0.012457600235939026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,127,0.01136159971356392
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,127,0.011857599765062333
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,255,0.012403199821710587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,255,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,511,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,511,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,1023,0.01053439974784851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,1023,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,2047,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,2047,0.010529600083827972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,4095,0.012638400495052337
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,4095,0.012452799826860428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,float16,8191,0.016582399606704712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,1,128,1,float16,fp8,8191,0.014614400267601014
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,1,0.01058880016207695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,1,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,3,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,3,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,7,0.01058719977736473
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,7,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,15,0.01050880029797554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,15,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,31,0.010550399869680404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,31,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,63,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,63,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,127,0.010468800365924836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,127,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,255,0.010555200278759003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,255,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,511,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,511,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,1023,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,1023,0.01077599972486496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,2047,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,2047,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,4095,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,4095,0.014545600116252898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,float16,8191,0.014654399454593658
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,1,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,1,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,2,128,1,float16,fp8,8191,0.01456640064716339
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,3,0.010334400087594986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,3,0.010371199995279311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,7,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,7,0.010316800326108932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,15,0.010361599922180175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,15,0.009046400338411332
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,31,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,31,0.010371199995279311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,63,0.010313600301742554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,63,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,127,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,127,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,255,0.010344000160694122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,255,0.010107199847698211
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,511,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,511,0.01034879982471466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,1023,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,1023,0.010284800082445145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,2047,0.010496000200510025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,2047,0.010500799864530563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,4095,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,4095,0.012411200255155564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,fp8,8191,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,4,128,1,float16,float16,8191,0.014630399644374847
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,1,0.008577600121498108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,1,0.008425600081682205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,3,0.008476799726486206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,3,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,7,0.008425600081682205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,7,0.008419200032949447
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,15,0.009593600034713745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,15,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,31,0.008444800227880477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,31,0.008564800024032593
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,63,0.008403199911117553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,63,0.008528000116348267
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,127,0.009201599657535553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,127,0.008476799726486206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,255,0.009542399644851684
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,255,0.008497600257396699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,511,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,511,0.00852160006761551
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,1023,0.008793599903583527
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,1023,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,2047,0.010526400059461594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,4095,0.012600000202655792
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,4095,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,2047,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,float16,8191,0.027193599939346315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,32,8,128,1,float16,fp8,8191,0.014486399292945863
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,1,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,1,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,3,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,3,0.01051200032234192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,7,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,7,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,15,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,15,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,31,0.010516799986362457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,31,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,63,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,63,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,127,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,127,0.010625600069761276
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,255,0.010662399977445603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,255,0.010569600015878677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,511,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,511,0.01255359947681427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,1023,0.01454080045223236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,1023,0.013980799913406372
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,2047,0.029135999083518983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,2047,0.016686399281024934
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,float16,4095,0.046942400932312014
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,1,128,1,float16,fp8,4095,0.030880001187324525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,1,0.012104000151157378
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,1,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,3,0.010564800351858139
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,3,0.011923199892044068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,7,0.011958400160074234
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,7,0.010513599961996078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,15,0.011777599900960922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,15,0.01053600013256073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,31,0.012457600235939026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,31,0.010521599650382995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,63,0.012387199699878693
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,63,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,127,0.012428800016641617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,127,0.010567999631166457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,255,0.012382400035858155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,511,0.014455999433994293
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,255,0.010761599987745285
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,511,0.0125231996178627
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,1023,0.02879360020160675
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,1023,0.016672000288963318
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,2047,0.04589279890060425
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,2047,0.02932479977607727
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,1,0.012567999958992004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,fp8,4095,0.04758400022983551
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,2,128,1,float16,float16,4095,0.08083999752998353
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,1,0.012380799651145935
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,3,0.012518399953842163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,7,0.012588800489902496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,3,0.012593600153923034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,7,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,15,0.012567999958992004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,31,0.012563200294971466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,15,0.012644800543785095
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,31,0.012427199631929398
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,63,0.012571200728416443
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,63,0.012615999579429627
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,127,0.012540799379348756
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,255,0.012556800246238708
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,127,0.012577599287033081
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,255,0.012611199915409089
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,511,0.02879680097103119
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,511,0.014507199823856353
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,1023,0.04686720073223114
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,1023,0.02949439883232117
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,2047,0.08217440247535705
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,2047,0.04733439981937408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,1,0.016777600347995757
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,float16,4095,0.15348800420761108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,4,128,1,float16,fp8,4095,0.08250240087509156
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,1,0.015904000401496886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,3,0.014532800018787383
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,3,0.016758400201797485
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,7,0.01668799966573715
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,15,0.01690559983253479
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,7,0.014537599682807923
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,15,0.016516800224781036
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,31,0.016836799681186676
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,31,0.014609600603580474
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,63,0.01751679927110672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,127,0.01666080057621002
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,63,0.01653439998626709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,127,0.016467200219631196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,255,0.027518400549888612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,255,0.01647839993238449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,511,0.04558559954166412
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,511,0.028865599632263185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,1023,0.08004000186920165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,1023,0.047672000527381894
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,2047,0.14884639978408815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,2047,0.082915198802948
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,1,0.013070400059223174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,1,0.012603199481964112
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,float16,4095,0.2864543914794922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,32,8,128,1,float16,fp8,4095,0.15964159965515137
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,3,0.013203200697898865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,3,0.012624000012874604
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,7,0.012604799866676331
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,7,0.012563200294971466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,31,0.012676799297332763
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,15,0.01404159963130951
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,15,0.012579199671745301
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,31,0.01247519999742508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,63,0.012556800246238708
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,63,0.0141744002699852
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,127,0.012774400413036346
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,127,0.012620800733566284
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,255,0.014452800154685974
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,255,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,511,0.01656319946050644
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,511,0.014561599493026734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,float16,1023,0.03059839904308319
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,1,128,1,float16,fp8,1023,0.01873279958963394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,1,0.014617599546909332
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,1,0.014795200526714325
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,3,0.01467519998550415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,3,0.012647999823093415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,7,0.014531199634075165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,7,0.013065600395202636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,15,0.014681600034236908
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,15,0.012503999471664428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,31,0.014571200311183929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,31,0.012651200592517852
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,63,0.014619199931621552
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,63,0.012788799405097962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,127,0.014694400131702423
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,127,0.012516799569129943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,255,0.014763200283050537
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,255,0.012620800733566284
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,511,0.01663679927587509
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,511,0.028963199257850646
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,float16,1023,0.047388800978660585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,1,0.016678400337696075
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,2,128,1,float16,fp8,1023,0.030985599756240843
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,3,0.016638399660587312
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,1,0.015609599649906158
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,3,0.014564800262451171
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,7,0.01656319946050644
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,7,0.01563359946012497
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,15,0.01658080071210861
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,31,0.018353599309921264
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,15,0.01465120017528534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,31,0.014606399834156037
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,63,0.018566399812698364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,63,0.014735999703407287
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,127,0.01823039948940277
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,255,0.016648000478744505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,127,0.01650079935789108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,255,0.027225598692893982
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,511,0.04531359970569611
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,511,0.028833600878715514
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,float16,1023,0.08042240142822266
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,1,0.02685759961605072
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,4,128,1,float16,fp8,1023,0.047137600183486936
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,1,0.022724799811840057
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,3,0.02686080038547516
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,3,0.022703999280929567
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,7,0.02682720124721527
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,7,0.022899200022220612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,15,0.026825600862503053
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,15,0.022711999714374542
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,31,0.026846399903297423
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,31,0.02279839962720871
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,63,0.02678399980068207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,63,0.02284799963235855
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,127,0.030932798981666565
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,127,0.022703999280929567
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,255,0.045592001080513
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,255,0.029660800099372865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,511,0.08112639784812928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,511,0.04763680100440979
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,1,0.018764799833297728
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,float16,1023,0.1497599959373474
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,32,8,128,1,float16,fp8,1023,0.08333439826965332
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,1,0.01855359971523285
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,3,0.018622399866580965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,3,0.018510399758815764
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,7,0.01874080002307892
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,15,0.01860480010509491
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,7,0.01751199960708618
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,15,0.018475200235843658
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,31,0.018619200587272643
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,31,0.01703200042247772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,63,0.01882079988718033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,63,0.018539200723171233
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,127,0.018721599876880646
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,127,0.018507200479507446
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,255,0.018566399812698364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,255,0.020396800339221956
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,float16,511,0.03187519907951355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,1,0.02273920029401779
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,1,128,1,float16,fp8,511,0.022720000147819518
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,3,0.022672000527381896
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,3,0.01860480010509491
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,1,0.01863519996404648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,7,0.02274080067873001
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,7,0.018628799915313722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,15,0.022732800245285033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,15,0.0186271995306015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,31,0.022756800055503845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,31,0.01852640062570572
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,63,0.02269120067358017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,127,0.022694399952888487
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,63,0.018646399676799773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,127,0.01860959976911545
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,255,0.028772801160812378
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,255,0.018654400110244752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,float16,511,0.047070398926734924
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,2,128,1,float16,fp8,511,0.03033599853515625
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,1,0.026846399903297423
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,1,0.024639999866485594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,3,0.026851201057434083
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,3,0.0247856006026268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,7,0.026817598938941957
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,7,0.02470560073852539
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,15,0.026862400770187377
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,15,0.024750399589538574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,31,0.0268640011548996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,31,0.024740800261497498
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,63,0.027008000016212463
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,63,0.024707199633121492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,127,0.031057599186897277
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,127,0.024710400402545928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,255,0.046460801362991334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,255,0.030377599596977233
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,float16,511,0.08219360113143921
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,4,128,1,float16,fp8,511,0.04761599898338318
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,1,0.04521760046482086
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,1,0.03897280097007751
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,3,0.04531520009040833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,7,0.045291200280189514
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,3,0.038729599118232726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,7,0.03904640078544617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,15,0.03901439905166626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,15,0.0455808013677597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,31,0.04536159932613373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,31,0.03903680145740509
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,63,0.03892160058021545
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,127,0.0513152003288269
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,63,0.04746719896793365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,127,0.0411327987909317
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,255,0.05040159821510315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,255,0.08225600123405456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,1,0.012379200011491776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,float16,511,0.15180000066757202
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,1,0.01239679977297783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,32,8,128,1,float16,fp8,511,0.08395839929580688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,3,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,3,0.012403199821710587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,7,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,7,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,15,0.012567999958992004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,15,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,31,0.012572799623012543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,31,0.012382400035858155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,63,0.012428800016641617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,63,0.012495999783277511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,127,0.012455999851226807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,127,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,255,0.012507200241088867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,255,0.012452799826860428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,511,0.012436799705028534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,511,0.012412799894809723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,1023,0.010515200346708298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,1023,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,2047,0.012547199428081513
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,2047,0.012409599870443344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,4095,0.014179199934005737
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,4095,0.014451199769973755
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,float16,8191,0.014604799449443817
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,1,0.010547199845314026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,1,128,1,float16,fp8,8191,0.014688000082969666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,1,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,3,0.01053759977221489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,3,0.010577599704265594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,7,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,15,0.010576000064611435
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,7,0.010468800365924836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,15,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,31,0.010632000118494033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,31,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,63,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,63,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,127,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,127,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,255,0.010556799918413162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,255,0.0105103999376297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,511,0.010489600151777268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,511,0.01051200032234192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,1023,0.010545600205659866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,1023,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,2047,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,4095,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,2047,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,float16,8191,0.016568000614643096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,4095,0.012531200051307678
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,2,128,1,float16,fp8,8191,0.014672000706195832
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,1,0.008488000184297562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,1,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,3,0.008414400368928909
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,3,0.00835679993033409
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,7,0.008448000252246856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,7,0.008537600189447403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,15,0.008337599784135818
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,31,0.008495999872684479
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,15,0.008588799834251403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,31,0.008379200100898742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,63,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,63,0.008472000062465668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,127,0.008588799834251403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,127,0.008534400165081025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,255,0.008475200086832047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,255,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,511,0.008449599891901017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,511,0.0084927998483181
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,1023,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,1023,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,2047,0.010542400181293488
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,2047,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,4095,0.012775999307632447
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,4095,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,float16,8191,0.027860799431800844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,4,128,1,float16,fp8,8191,0.014560000598430633
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,1,0.008537600189447403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,1,0.010022400319576264
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,3,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,3,0.008551999926567078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,7,0.010316800326108932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,7,0.008416000008583068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,15,0.010209599882364273
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,15,0.008500800281763077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,31,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,31,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,63,0.010332799702882766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,127,0.010384000092744827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,63,0.008755200356245042
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,127,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,255,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,255,0.00865280032157898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,511,0.01035040020942688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,511,0.010337600111961364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,1023,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,2047,0.01252799928188324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,1023,0.010814400017261505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,2047,0.010566399991512298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,4095,0.02826719880104065
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,4095,0.014691199362277984
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,fp8,8191,0.02876160144805908
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,32,8,128,1,float16,float16,8191,0.045289599895477296
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,1,0.03107360005378723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,1,0.028799998760223388
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,3,0.030828800797462464
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,3,0.028857600688934327
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,7,0.030836799740791322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,7,0.0287663996219635
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,15,0.028814399242401124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,15,0.030947199463844298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,31,0.03099679946899414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,31,0.028803199529647827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,63,0.028870400786399842
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,63,0.031088000535964964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,127,0.030982398986816408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,127,0.02882080078125
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,float16,255,0.03424479961395264
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,1,128,1,float16,fp8,255,0.02898559868335724
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,1,0.03704800009727478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,1,0.028960001468658448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,3,0.037099200487136844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,3,0.028830400109291075
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,15,0.03710559904575348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,7,0.028839999437332155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,7,0.03723360002040863
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,15,0.028758400678634645
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,31,0.0369951993227005
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,31,0.028998398780822755
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,63,0.03711999952793121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,63,0.028718400001525878
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,127,0.028774398565292358
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,255,0.04832479953765869
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,float16,127,0.03903039991855621
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,2,128,1,float16,fp8,255,0.03283840119838714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,1,0.04535520076751709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,1,0.03925279974937439
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,3,0.04535360038280487
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,3,0.039233601093292235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,7,0.04542079865932465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,7,0.03915199935436249
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,15,0.04524959921836853
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,15,0.0392304003238678
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,31,0.04713920056819916
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,31,0.03917919993400574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,63,0.0392655998468399
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,63,0.047537600994110106
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,127,0.05155040025711059
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,127,0.04149119853973389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,fp8,255,0.05084159970283508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,4,128,1,float16,float16,255,0.0833952009677887
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,1,0.08216480016708375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,1,0.06981279850006103
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,3,0.08227360248565674
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,3,0.06798239946365356
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,7,0.08223999738693237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,7,0.069896000623703
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,15,0.08230080008506775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,15,0.06981599926948548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,31,0.08431199789047242
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,31,0.0678160011768341
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,63,0.08450880050659179
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,63,0.07006400227546691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,127,0.09168800115585327
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,127,0.07214879989624023
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,fp8,255,0.09122880101203919
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,32,8,128,1,float16,float16,255,0.15514400005340576
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,1,0.05148000121116638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,1,0.049348801374435425
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,3,0.05307360291481018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,3,0.04787200093269348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,7,0.05249599814414978
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,15,0.05138720273971557
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,15,0.04857119917869568
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,7,0.049527999758720395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,31,0.04901759922504425
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,31,0.051798397302627565
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,63,0.05581439733505249
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,63,0.04866879880428314
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,float16,127,0.0578544020652771
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,1,128,1,float16,fp8,127,0.0497296005487442
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,1,0.04729120135307312
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,1,0.06578720211982728
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,3,0.047270399332046506
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,7,0.06573439836502075
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,3,0.06584479808807372
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,7,0.047224000096321106
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,15,0.06581119894981384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,15,0.04761599898338318
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,31,0.06585919857025146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,31,0.0474128007888794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,63,0.04741120040416717
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,63,0.06673920154571533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,float16,127,0.06741440296173096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,2,128,1,float16,fp8,127,0.049404799938201904
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,1,0.08416159749031067
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,1,0.07192639708518982
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,3,0.08224800229072571
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,3,0.07135519981384278
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,7,0.08374239802360535
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,7,0.07179679870605468
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,31,0.08437119722366333
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,15,0.08445280194282531
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,15,0.07193440198898315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,31,0.07202079892158508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,63,0.08494560122489929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,63,0.07223359942436218
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,float16,127,0.09321280121803284
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,1,0.15618079900741577
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,4,128,1,float16,fp8,127,0.07450399994850158
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,1,0.1295632004737854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,3,0.15610079765319823
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,3,0.12963839769363403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,7,0.15796799659729005
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,7,0.12940479516983033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,15,0.13124159574508668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,15,0.1581712007522583
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,31,0.15820159912109374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,31,0.1312175989151001
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,63,0.15917760133743286
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,63,0.131713604927063
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,1,0.012415999919176102
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,float16,127,0.1711951971054077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,1,0.01250240057706833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,32,8,128,1,float16,fp8,127,0.13581600189208984
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,3,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,3,0.012415999919176102
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,7,0.012566399574279786
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,7,0.012371200323104858
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,15,0.012436799705028534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,15,0.012590399384498597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,31,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,31,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,63,0.012542399764060973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,63,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,127,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,127,0.012611199915409089
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,255,0.012404800206422806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,255,0.01234079971909523
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,511,0.012433599680662155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,511,0.012532800436019897
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,1023,0.012358400225639343
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,1023,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,2047,0.012401600182056428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,2047,0.012615999579429627
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,4095,0.014478400349617004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,float16,8191,0.016684800386428833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,8191,0.016164800524711607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,1,128,1,float16,fp8,4095,0.014443199336528777
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,1,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,1,0.010363200306892395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,3,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,3,0.010780800133943558
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,7,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,7,0.010311999917030334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,15,0.010363200306892395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,15,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,31,0.010334400087594986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,31,0.010337600111961364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,63,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,63,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,255,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,127,0.01069760024547577
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,127,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,511,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,255,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,511,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,1023,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,1023,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,2047,0.012489599734544754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,2047,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,4095,0.013411200046539307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,8191,0.014903999865055084
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,float16,8191,0.028169599175453187
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,2,128,1,float16,fp8,4095,0.012611199915409089
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,1,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,1,0.008414400368928909
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,3,0.010171200335025787
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,3,0.008395200222730636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,7,0.008403199911117553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,7,0.0084927998483181
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,15,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,15,0.008436799794435502
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,31,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,31,0.008433599770069123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,63,0.008955200016498566
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,63,0.008798400312662125
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,127,0.008856000006198883
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,127,0.008433599770069123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,255,0.010342399775981902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,255,0.00835840031504631
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,511,0.008392000198364257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,511,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,1023,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,1023,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,2047,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,2047,0.012598399817943574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,4095,0.027321600914001466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,4095,0.013353599607944489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,fp8,8191,0.027871999144554137
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,1,0.008427199721336365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,4,128,1,float16,float16,8191,0.04551360011100769
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,1,0.00840959995985031
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,3,0.008367999643087386
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,3,0.008479999750852585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,7,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,7,0.008364800363779068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,15,0.008481600135564805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,31,0.008433599770069123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,31,0.008510400354862214
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,15,0.008404800295829773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,63,0.008475200086832047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,63,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,127,0.008473599702119828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,127,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,255,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,255,0.008528000116348267
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,511,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,511,0.00979520007967949
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,1023,0.012383999675512314
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,1023,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,2047,0.026787200570106508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,2047,0.012590399384498597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,4095,0.044619199633598325
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,4095,0.0270224004983902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,float16,8191,0.07910559773445129
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,1,0.010471999645233154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,32,8,128,1,float16,fp8,8191,0.04525440037250519
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,1,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,3,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,3,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,7,0.010548800230026245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,7,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,15,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,15,0.010507199913263321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,31,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,31,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,63,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,127,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,63,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,127,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,255,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,255,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,511,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,511,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,1023,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,1023,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,2047,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,2047,0.01249760016798973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,4095,0.014502400159835815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,4095,0.028857600688934327
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,8191,0.04585280120372772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,8191,0.02887679934501648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,fp8,16383,0.04707199931144714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,1,128,1,float16,float16,16383,0.08002240061759949
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,1,0.008342400193214417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,3,0.008422400057315826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,3,0.00835679993033409
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,1,0.008380799740552902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,7,0.008500800281763077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,15,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,7,0.008342400193214417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,15,0.008504000306129456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,31,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,31,0.008444800227880477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,63,0.008420799672603608
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,63,0.008507200330495835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,127,0.00846880003809929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,127,0.008495999872684479
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,255,0.00843999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,255,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,511,0.010753600299358368
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,511,0.010329599678516387
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,1023,0.01242239996790886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,1023,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,2047,0.027046400308609008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,2047,0.012518399953842163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,4095,0.04465279877185822
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,4095,0.027559998631477355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,8191,0.07880160212516785
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,8191,0.04526239931583405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,float16,16383,0.14689120054244995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,2,128,1,float16,fp8,16383,0.08013439774513245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,1,0.010326399654150008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,1,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,3,0.01011039987206459
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,3,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,7,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,7,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,15,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,15,0.010384000092744827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,31,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,31,0.011224000155925751
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,63,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,63,0.010339199751615524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,127,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,127,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,255,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,255,0.010313600301742554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,511,0.012503999471664428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,511,0.010675200074911118
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,1023,0.027599999308586122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,1023,0.014451199769973755
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,2047,0.0452208012342453
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,2047,0.02816160023212433
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,4095,0.07992320060729981
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,4095,0.04591520130634308
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,8191,0.08187360167503357
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,8191,0.14955519437789916
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,1,0.012598399817943574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,float16,16383,0.2895776033401489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,1,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,4,128,1,float16,fp8,16383,0.15303679704666137
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,3,0.012577599287033081
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,3,0.01236959993839264
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,7,0.012630400061607362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,7,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,15,0.012624000012874604
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,15,0.012379200011491776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,31,0.012515200674533844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,31,0.01239359974861145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,63,0.012516799569129943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,63,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,127,0.012598399817943574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,255,0.01249760016798973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,127,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,255,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,511,0.027635198831558228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,511,0.014724799990653991
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,1023,0.04556480050086975
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,1023,0.028867200016975403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,2047,0.08201119899749756
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,2047,0.04735519886016846
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,4095,0.15225280523300172
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,4095,0.08458080291748046
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,8191,0.2946320056915283
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,8191,0.15924479961395263
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,1,0.010468800365924836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,1,0.010292799770832061
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,3,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,3,0.009537599980831146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,float16,16383,0.5785903930664062
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,7,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,15,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,16,8,128,1,float16,fp8,16383,0.31435039043426516
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,15,0.009652800112962722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,7,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,31,0.008419200032949447
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,31,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,63,0.010361599922180175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,63,0.009040000289678574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,127,0.010345599800348281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,127,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,255,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,511,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,255,0.010558400303125381
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,511,0.010361599922180175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,1023,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,1023,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,2047,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,4095,0.012433599680662155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,2047,0.010744000226259232
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,4095,0.012411200255155564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,8191,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,float16,16383,0.016761599481105803
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,8191,0.012732799351215362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,1,128,1,float16,fp8,16383,0.0185248002409935
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,1,0.00844319984316826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,3,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,1,0.008644799888134002
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,3,0.008535999804735184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,7,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,7,0.008577600121498108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,15,0.008508799970149994
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,15,0.00843999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,63,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,31,0.008516799658536911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,31,0.01079839989542961
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,63,0.008515200018882752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,127,0.00868159979581833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,127,0.00921280011534691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,255,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,255,0.008535999804735184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,511,0.010516799986362457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,511,0.010326399654150008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,1023,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,2047,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,1023,0.008584000170230865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,2047,0.010371199995279311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,4095,0.010548800230026245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,4095,0.01056160032749176
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,8191,0.012646399438381195
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,8191,0.01441120058298111
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,float16,16383,0.016764800250530242
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,2,128,1,float16,fp8,16383,0.01674720048904419
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,1,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,3,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,3,0.009990400075912476
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,1,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,7,0.010328000038862228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,7,0.010331200063228607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,15,0.008537600189447403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,15,0.010342399775981902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,31,0.010361599922180175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,63,0.010313600301742554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,31,0.008515200018882752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,63,0.008684799820184708
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,127,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,255,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,127,0.008529599756002426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,255,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,511,0.010468800365924836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,511,0.008489599823951722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,1023,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,1023,0.00902400016784668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,2047,0.010496000200510025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,2047,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,4095,0.012337599694728852
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,8191,0.01443679928779602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,4095,0.01058880016207695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,8191,0.014431999623775482
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,float16,16383,0.016444799304008485
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,4,128,1,float16,fp8,16383,0.014620800316333771
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,1,0.010883200168609618
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,1,0.010264000296592713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,3,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,3,0.008736000210046769
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,7,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,7,0.008448000252246856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,15,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,15,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,31,0.010513599961996078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,31,0.010337600111961364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,63,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,63,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,127,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,127,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,255,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,511,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,255,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,1023,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,511,0.008591999858617782
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,1023,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,2047,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,2047,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,4095,0.012436799705028534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,4095,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,8191,0.01406719982624054
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,float16,16383,0.016523200273513793
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,8191,0.012643200159072877
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,1,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,1,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,16,8,128,1,float16,fp8,16383,0.014590400457382201
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,3,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,3,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,7,0.010619200021028518
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,7,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,15,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,15,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,31,0.010471999645233154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,31,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,63,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,63,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,127,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,127,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,255,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,255,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,511,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,511,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,1023,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,1023,0.010553599894046783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,2047,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,2047,0.010635200142860412
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,4095,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,4095,0.012430399656295776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,8191,0.01454080045223236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,8191,0.014468799531459808
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,float16,16383,0.018692800402641298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,1,128,1,float16,fp8,16383,0.018540799617767334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,1,0.010313600301742554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,1,0.008556800335645676
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,3,0.009055999666452407
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,3,0.008510400354862214
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,7,0.009854400157928466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,7,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,15,0.009998399764299393
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,15,0.008432000130414962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,31,0.008427199721336365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,31,0.008523199707269669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,63,0.008483199775218964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,63,0.008619199693202972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,127,0.008484800159931184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,127,0.01032319962978363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,255,0.008432000130414962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,255,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,511,0.009015999734401703
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,511,0.010345599800348281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,1023,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,1023,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,2047,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,2047,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,4095,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,4095,0.011102399975061416
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,8191,0.014494399726390838
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,8191,0.014406399428844452
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,float16,16383,0.0166143998503685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,2,128,1,float16,fp8,16383,0.015833599865436553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,1,0.010371199995279311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,3,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,1,0.00873280018568039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,3,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,7,0.010471999645233154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,15,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,7,0.00846719965338707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,15,0.008664000034332275
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,31,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,63,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,31,0.008427199721336365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,63,0.008425600081682205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,127,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,127,0.009848000109195709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,255,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,255,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,511,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,1023,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,1023,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,511,0.008479999750852585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,2047,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,2047,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,4095,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,4095,0.01242239996790886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,8191,0.014382399618625641
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,float16,16383,0.01604959964752197
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,8191,0.01265919953584671
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,4,128,1,float16,fp8,16383,0.01454399973154068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,1,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,1,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,3,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,3,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,7,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,7,0.010516799986362457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,15,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,15,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,31,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,31,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,63,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,63,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,127,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,127,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,255,0.010566399991512298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,255,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,511,0.010292799770832061
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,511,0.010115200281143188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,1023,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,1023,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,2047,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,4095,0.012591999769210816
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,2047,0.010494399815797806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,8191,0.014617599546909332
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,4095,0.01244639977812767
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,8191,0.012529599666595458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,float16,16383,0.030055999755859375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,16,8,128,1,float16,fp8,16383,0.015855999290943147
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,1,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,1,0.01034879982471466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,3,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,3,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,7,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,7,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,15,0.010492800176143647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,31,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,15,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,31,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,63,0.01064639985561371
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,63,0.010313600301742554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,127,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,255,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,127,0.0108255997300148
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,255,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,511,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,1023,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,511,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,1023,0.010302399843931198
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,2047,0.011929599940776825
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,2047,0.010494399815797806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,4095,0.01268640011548996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,4095,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,8191,0.027644801139831542
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,8191,0.015761600434780122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,float16,16383,0.04524640142917633
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,1,128,1,float16,fp8,16383,0.0294048011302948
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,1,0.00841279998421669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,3,0.010320000350475311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,1,0.010531199723482132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,3,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,7,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,15,0.008401600271463394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,7,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,15,0.008745600283145905
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,31,0.008433599770069123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,63,0.010384000092744827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,31,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,127,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,63,0.008392000198364257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,127,0.010328000038862228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,255,0.010320000350475311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,255,0.008387199789285659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,511,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,511,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,1023,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,1023,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,2047,0.013059200346469879
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,2047,0.010542400181293488
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,4095,0.027587199211120607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,4095,0.014379200339317322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,8191,0.02831200063228607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,8191,0.04576640129089356
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,1,0.008483199775218964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,float16,16383,0.07942879796028138
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,2,128,1,float16,fp8,16383,0.045505601167678836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,1,0.008403199911117553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,3,0.008488000184297562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,3,0.008604799956083297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,7,0.008353599905967712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,7,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,15,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,15,0.008542399853467941
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,31,0.009121599793434142
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,31,0.008408000320196151
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,63,0.00843999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,63,0.008929599821567536
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,127,0.008416000008583068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,127,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,255,0.008948799967765809
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,255,0.008448000252246856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,511,0.010300800204277039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,511,0.010315199941396713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,1023,0.010518400371074677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,1023,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,2047,0.026924800872802735
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,2047,0.012559999525547028
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,4095,0.04434399902820587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,4095,0.027263998985290527
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,8191,0.07916799783706666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,8191,0.045265600085258484
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,1,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,float16,16383,0.14846240282058715
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,1,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,4,128,1,float16,fp8,16383,0.07971199750900268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,3,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,3,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,7,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,7,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,15,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,15,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,31,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,31,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,63,0.010566399991512298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,63,0.01055999994277954
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,127,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,127,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,255,0.010489600151777268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,255,0.009916800260543823
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,511,0.012619200348854064
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,511,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,1023,0.027451199293136597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,1023,0.014206400513648987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,2047,0.045177599787712096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,2047,0.02791680097579956
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,4095,0.08009920120239258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,4095,0.04646559953689575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,8191,0.14945600032806397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,8191,0.08333119750022888
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,fp8,16383,0.1599951982498169
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,16,8,128,1,float16,float16,16383,0.2872879981994629
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,1,0.010582400113344192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,1,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,7,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,3,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,3,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,7,0.010625600069761276
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,15,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,15,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,31,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,31,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,63,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,63,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,127,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,127,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,255,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,511,0.010496000200510025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,255,0.010468800365924836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,1023,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,1023,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,511,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,2047,0.012505599856376648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,4095,0.01249760016798973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,2047,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,4095,0.01252480000257492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,8191,0.014519999921321868
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,8191,0.014575999975204468
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,fp8,16383,0.016616000235080718
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,1,128,1,float16,float16,16383,0.016675199568271636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,1,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,1,0.008481600135564805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,3,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,3,0.00973920002579689
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,7,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,7,0.008427199721336365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,15,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,31,0.010335999727249145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,15,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,31,0.008387199789285659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,63,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,63,0.008449599891901017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,127,0.00982080027461052
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,255,0.009431999921798707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,127,0.009484799951314926
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,255,0.008376000076532364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,511,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,511,0.009712000191211701
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,1023,0.01035040020942688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,1023,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,2047,0.010384000092744827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,2047,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,4095,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,4095,0.010740800201892853
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,8191,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,8191,0.012510399520397186
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,fp8,16383,0.014398400485515595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,2,128,1,float16,float16,16383,0.016572800278663636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,1,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,1,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,3,0.010326399654150008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,3,0.00979520007967949
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,7,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,7,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,15,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,15,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,31,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,31,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,63,0.010342399775981902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,63,0.009286399930715561
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,127,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,127,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,255,0.010526400059461594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,511,0.010468800365924836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,255,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,511,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,1023,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,1023,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,2047,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,2047,0.010532800108194351
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,4095,0.012507200241088867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,4095,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,8191,0.012464000284671784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,8191,0.01467359960079193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,float16,16383,0.029867199063301087
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,1,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,1,0.008481600135564805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,4,128,1,float16,fp8,16383,0.016572800278663636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,3,0.008472000062465668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,3,0.0083856001496315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,7,0.008550400286912918
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,7,0.008399999886751174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,15,0.009344000369310379
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,15,0.008486399799585343
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,31,0.0095551997423172
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,31,0.008382400125265121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,63,0.008436799794435502
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,63,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,127,0.008459199965000153
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,127,0.008516799658536911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,255,0.008416000008583068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,255,0.008504000306129456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,511,0.008345600217580795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,511,0.008481600135564805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,1023,0.00843999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,1023,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,2047,0.010552000254392624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,2047,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,4095,0.012651200592517852
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,4095,0.01053600013256073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,8191,0.014457599818706512
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,8191,0.027039998769760133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,float16,16383,0.04484640061855316
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,16,8,128,1,float16,fp8,16383,0.027668800950050355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,1,0.008499199897050858
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,1,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,3,0.010543999820947647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,3,0.008404800295829773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,7,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,15,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,15,0.008646400272846222
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,7,0.008753599971532822
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,31,0.010292799770832061
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,31,0.008975999802350998
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,63,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,63,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,127,0.010344000160694122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,127,0.008374399691820144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,255,0.009247999638319016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,255,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,511,0.010480000078678131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,1023,0.012398400157690049
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,1023,0.012865599989891053
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,511,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,2047,0.027702400088310243
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,4095,0.045407998561859134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,2047,0.014787200093269347
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,4095,0.028830400109291075
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,1,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,float16,8191,0.07997440099716187
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,1,128,1,float16,fp8,8191,0.04568960070610047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,1,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,3,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,3,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,7,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,7,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,15,0.01067200005054474
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,15,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,31,0.010593599826097488
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,63,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,63,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,31,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,127,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,127,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,255,0.010678400099277497
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,255,0.010329599678516387
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,511,0.012582400441169738
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,1023,0.026919999718666078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,1023,0.01446399986743927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,511,0.012457600235939026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,2047,0.045265600085258484
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,2047,0.02887200117111206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,4095,0.08008319735527039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,4095,0.047004801034927365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,1,0.012521600723266602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,fp8,8191,0.0821008026599884
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,1,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,2,128,1,float16,float16,8191,0.14828319549560548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,3,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,3,0.012639999389648438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,7,0.012678399682044983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,15,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,7,0.012427199631929398
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,15,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,31,0.012683199346065521
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,31,0.01252799928188324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,63,0.012639999389648438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,63,0.012399999797344208
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,127,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,255,0.012628799676895142
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,127,0.012449599802494049
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,255,0.01242239996790886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,511,0.0284496009349823
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,511,0.01467680037021637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,1023,0.04657439887523651
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,1023,0.028916800022125246
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,2047,0.08194400072097778
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,2047,0.04731679856777191
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,4095,0.15243200063705445
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,4095,0.08268799781799316
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,fp8,8191,0.1535423994064331
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,4,128,1,float16,float16,8191,0.29447839260101316
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,1,0.01669439971446991
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,1,0.01650719940662384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,3,0.01671839952468872
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,3,0.016164800524711607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,7,0.016961599886417388
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,7,0.016547200083732606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,15,0.016524800658226015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,31,0.01713760048151016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,15,0.01683039963245392
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,31,0.016553600132465363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,63,0.017407999932765962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,63,0.01475519984960556
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,127,0.017350399494171144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,127,0.01648319959640503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,255,0.016155199706554414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,511,0.045177599787712096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,255,0.027452799677848815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,511,0.02887519896030426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,1023,0.07991039752960205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,1023,0.04628799855709076
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,2047,0.0822928011417389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,2047,0.1483407974243164
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,4095,0.28597440719604494
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,4095,0.15689599514007568
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,1,0.011740799993276596
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,1,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,3,0.012052799761295318
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,3,0.010518400371074677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,fp8,8191,0.3096992015838623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,16,8,128,1,float16,float16,8191,0.5620528221130371
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,7,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,15,0.010543999820947647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,7,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,15,0.012280000001192093
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,31,0.012569600343704223
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,31,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,63,0.012324800342321396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,63,0.010516799986362457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,127,0.01242239996790886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,127,0.010529600083827972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,255,0.011737599968910217
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,255,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,511,0.014454400539398194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,511,0.012639999389648438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,1023,0.028233599662780762
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,1023,0.016097599267959596
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,float16,2047,0.045423999428749084
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,1,128,1,float16,fp8,2047,0.028883200883865357
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,1,0.01257600039243698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,3,0.012555199861526489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,1,0.012382400035858155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,3,0.012399999797344208
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,7,0.012531200051307678
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,7,0.012515200674533844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,15,0.012678399682044983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,15,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,31,0.012558400630950928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,31,0.012438400089740754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,63,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,63,0.012342400103807449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,127,0.012580800056457519
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,127,0.012457600235939026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,255,0.012582400441169738
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,255,0.012470400333404541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,511,0.014703999459743499
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,511,0.027963200211524965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,1023,0.04711999893188477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,1023,0.02956640124320984
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,float16,2047,0.08115360140800476
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,1,0.017531199753284453
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,2,128,1,float16,fp8,2047,0.04775519967079163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,1,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,3,0.01669600009918213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,3,0.014659200608730317
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,7,0.016628800332546233
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,7,0.014585599303245544
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,15,0.016705599427223206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,15,0.014611199498176575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,31,0.016689600050449373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,31,0.014735999703407287
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,63,0.016654400527477263
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,63,0.014897599816322327
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,127,0.016732800006866454
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,127,0.014595200121402741
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,255,0.027449598908424376
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,255,0.01653279960155487
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,511,0.04529280066490173
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,511,0.028790399432182312
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,1023,0.0469296008348465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,1023,0.08049920201301575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,1,0.027091199159622194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,float16,2047,0.1493391990661621
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,4,128,1,float16,fp8,2047,0.08206560015678406
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,1,0.022843199968338012
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,3,0.026782399415969847
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,7,0.02685439884662628
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,3,0.022889600694179536
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,7,0.022806400060653688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,15,0.026811200380325317
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,15,0.02282239943742752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,31,0.026846399903297423
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,31,0.022767999768257143
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,63,0.0268640011548996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,127,0.030928000807762146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,127,0.022787199914455415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,63,0.022940799593925476
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,255,0.04537599980831146
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,255,0.03019840121269226
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,511,0.047486400604248045
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,511,0.08019520044326782
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,1023,0.14910240173339845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,1023,0.0825215995311737
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,1,0.01467680037021637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,1,0.014484800398349762
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,fp8,2047,0.15495840311050416
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,16,8,128,1,float16,float16,2047,0.28775360584259035
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,3,0.014606399834156037
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,3,0.01443839967250824
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,7,0.013371199369430542
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,15,0.014612799882888794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,7,0.014772799611091614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,15,0.01313599944114685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,31,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,63,0.014716799557209014
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,31,0.014267200231552124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,63,0.014433600008487701
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,127,0.014703999459743499
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,255,0.01488959938287735
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,127,0.014531199634075165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,255,0.014484800398349762
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,511,0.028828799724578857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,511,0.01679839938879013
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,float16,1023,0.04755840003490448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,1,128,1,float16,fp8,1023,0.03059200048446655
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,1,0.016564799845218657
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,1,0.01680160015821457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,3,0.016574400663375854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,3,0.016462400555610657
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,7,0.016847999393939973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,7,0.016371199488639833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,15,0.01663679927587509
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,31,0.01661120057106018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,15,0.016710400581359863
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,31,0.0165583997964859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,63,0.016631999611854555
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,127,0.016646400094032288
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,63,0.016663999855518342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,127,0.016467200219631196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,255,0.02844479978084564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,511,0.04533919990062714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,511,0.028863999247550964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,255,0.015919999778270723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,1,0.02677600085735321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,float16,1023,0.08012319803237915
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,2,128,1,float16,fp8,1023,0.04732959866523743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,1,0.02276960015296936
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,3,0.026800000667572023
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,3,0.02282879948616028
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,7,0.02682879865169525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,7,0.02276639938354492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,15,0.026900801062583923
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,15,0.022771200537681578
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,31,0.026859200000762938
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,31,0.022899200022220612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,63,0.02688960134983063
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,63,0.02279040068387985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,127,0.030895999073982237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,127,0.02292319983243942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,255,0.0460671991109848
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,255,0.029707199335098265
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,511,0.08167679905891419
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,511,0.0472927987575531
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,1,0.04527679979801178
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,float16,1023,0.15026079416275023
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,4,128,1,float16,fp8,1023,0.08223040103912353
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,1,0.03913280069828033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,3,0.04528000056743622
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,7,0.04527359902858734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,3,0.0391072005033493
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,7,0.03900479972362518
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,15,0.04524959921836853
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,15,0.039113599061965945
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,31,0.04531840085983276
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,31,0.0391184002161026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,63,0.04737119972705841
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,63,0.03916000127792359
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,127,0.05035840272903443
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,127,0.04122720062732697
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,255,0.08174399733543396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,255,0.04977119863033295
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,511,0.14980319738388062
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,511,0.08330879807472229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,1,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,float16,1023,0.28695199489593504
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,1,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,16,8,128,1,float16,fp8,1023,0.15410399436950684
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,3,0.010590399801731109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,3,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,7,0.01061599999666214
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,7,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,15,0.010543999820947647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,15,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,31,0.010542400181293488
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,63,0.010572800040245056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,31,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,127,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,63,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,255,0.010547199845314026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,127,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,255,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,511,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,1023,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,511,0.01053759977221489
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,1023,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,2047,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,2047,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,4095,0.01446399986743927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,4095,0.012591999769210816
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,8191,0.014577600359916686
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,8191,0.014483200013637542
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,fp8,16383,0.016550399363040924
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,1,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,1,128,1,float16,float16,16383,0.01675360053777695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,3,0.010316800326108932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,1,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,3,0.009692800045013428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,7,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,7,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,15,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,15,0.009243199974298478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,31,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,31,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,63,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,63,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,127,0.010328000038862228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,127,0.009567999839782714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,255,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,255,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,511,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,511,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,1023,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,2047,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,1023,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,4095,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,2047,0.010564800351858139
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,4095,0.012411200255155564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,8191,0.01454080045223236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,float16,16383,0.03017440140247345
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,8191,0.014137600362300873
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,2,128,1,float16,fp8,16383,0.01653439998626709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,1,0.010190399736166
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,1,0.008575999736785888
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,3,0.008473599702119828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,3,0.008502399921417237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,7,0.008388800173997879
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,15,0.010278400033712387
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,7,0.009484799951314926
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,15,0.0083856001496315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,31,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,31,0.008505599945783615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,63,0.008414400368928909
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,63,0.008667200058698653
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,127,0.008747199922800064
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,127,0.008387199789285659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,255,0.008551999926567078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,255,0.008497600257396699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,511,0.008404800295829773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,511,0.008553600311279297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,1023,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,1023,0.0095040000975132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,2047,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,4095,0.012516799569129943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,2047,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,4095,0.010639999806880952
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,8191,0.026966398954391478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,8191,0.014470399916172027
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,float16,16383,0.04531840085983276
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,4,128,1,float16,fp8,16383,0.02715040147304535
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,1,0.009673599898815156
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,1,0.008524800091981888
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,3,0.008526399731636047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,7,0.010225600004196167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,3,0.00846719965338707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,7,0.008382400125265121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,15,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,31,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,15,0.008505599945783615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,31,0.010334400087594986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,63,0.010345599800348281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,63,0.008529599756002426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,127,0.010318399965763092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,127,0.009068799763917923
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,255,0.009647999703884125
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,255,0.010305599868297577
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,511,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,511,0.010310400277376175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,1023,0.010844799876213073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,1023,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,2047,0.012604799866676331
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,2047,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,4095,0.027057600021362305
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,4095,0.014508800208568573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,8191,0.02797600030899048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,8191,0.04521119892597199
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,float16,16383,0.07906079888343812
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,16,8,128,1,float16,fp8,16383,0.04547199904918671
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,1,0.018692800402641298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,1,0.02271520048379898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,3,0.022623999416828154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,3,0.018596799671649934
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,7,0.0227183997631073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,7,0.018596799671649934
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,15,0.022646400332450866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,15,0.018606400489807128
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,31,0.022785599529743194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,31,0.01865279972553253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,63,0.022697600722312927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,63,0.01855040043592453
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,127,0.022724799811840057
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,127,0.018654400110244752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,255,0.028907200694084166
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,255,0.01868640035390854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,fp8,511,0.030817601084709167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,1,128,1,float16,float16,511,0.04734880030155182
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,1,0.026833599805831908
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,1,0.024689599871635437
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,3,0.026804798841476442
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,3,0.024779200553894043
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,7,0.026998400688171387
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,7,0.024736000597476958
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,15,0.026819199323654175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,15,0.02475679963827133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,31,0.02683520019054413
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,31,0.024676799774169922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,63,0.026940798759460448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,63,0.024792000651359558
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,127,0.030806401371955873
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,127,0.02476319968700409
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,255,0.03039200007915497
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,255,0.0470768004655838
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,float16,511,0.08221439719200134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,2,128,1,float16,fp8,511,0.047521600127220155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,1,0.045396798849105836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,3,0.04528000056743622
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,1,0.03899520039558411
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,7,0.045239999890327454
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,7,0.03918560147285462
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,3,0.03934879899024964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,15,0.045244801044464114
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,15,0.03907040059566498
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,31,0.045244801044464114
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,31,0.039084801077842714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,63,0.04746719896793365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,63,0.03909280002117157
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,127,0.05127040147781372
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,127,0.04129599928855896
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,255,0.08216320276260376
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,255,0.049435201287269595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,1,0.08212479948997498
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,float16,511,0.15210720300674438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,4,128,1,float16,fp8,511,0.08346080183982849
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,1,0.06982399821281433
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,3,0.06987839937210083
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,3,0.08226720094680787
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,7,0.08214399814605713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,7,0.069896000623703
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,15,0.08214880228042602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,15,0.0698751986026764
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,31,0.08432480096817016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,31,0.06988639831542968
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,63,0.0842736005783081
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,63,0.0711903989315033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,127,0.09040160179138183
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,127,0.0720416009426117
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,255,0.08987680077552795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,255,0.15198400020599365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,1,0.037057599425315856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,float16,511,0.2891760110855103
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,1,0.028915199637413024
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,16,8,128,1,float16,fp8,511,0.1550447940826416
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,3,0.02892000079154968
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,3,0.03705599904060364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,15,0.03702560067176819
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,7,0.03701600134372711
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,7,0.02890399992465973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,15,0.028891199827194215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,31,0.037083199620246886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,31,0.02890079915523529
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,63,0.03711360096931458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,63,0.02911359965801239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,127,0.0390751987695694
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,127,0.02885119915008545
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,float16,255,0.04807359874248505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,1,128,1,float16,fp8,255,0.03288640081882477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,1,0.045219200849533084
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,3,0.045342400670051575
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,1,0.039211198687553406
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,3,0.04129759967327118
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,7,0.03980959951877594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,15,0.04532319903373718
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,7,0.045552000403404236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,31,0.04663839936256409
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,15,0.04060640037059784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,63,0.0472815990447998
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,63,0.04105919897556305
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,31,0.041289600729942325
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,127,0.05119839906692505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,127,0.0434255987405777
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,float16,255,0.0842960000038147
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,2,128,1,float16,fp8,255,0.05138559937477112
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,1,0.08212159872055054
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,3,0.08208640217781067
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,1,0.07027519941329956
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,3,0.06980159878730774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,7,0.08207839727401733
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,15,0.08233280181884765
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,7,0.07007840275764465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,15,0.0698848009109497
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,31,0.08440960049629212
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,31,0.07013919949531555
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,63,0.08425440192222595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,63,0.07195360064506531
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,127,0.0917680025100708
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,127,0.07382400035858154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,float16,255,0.15535839796066284
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,1,0.155841600894928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,4,128,1,float16,fp8,255,0.09006879925727844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,1,0.12943520545959472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,3,0.1560479998588562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,3,0.13143680095672608
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,7,0.15624959468841554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,7,0.13127039670944213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,15,0.1580672025680542
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,31,0.15794719457626344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,15,0.13134080171585083
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,31,0.1314895987510681
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,63,0.13141920566558837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,63,0.158188796043396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,127,0.17048319578170776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,127,0.13526560068130494
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,1,0.010550399869680404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,float16,255,0.29475040435791017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,1,0.010494399815797806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,3,0.01034879982471466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,16,8,128,1,float16,fp8,255,0.17279200553894042
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,3,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,7,0.010547199845314026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,15,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,15,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,7,0.010550399869680404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,31,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,31,0.010361599922180175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,63,0.010556799918413162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,63,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,127,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,127,0.010449600219726563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,255,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,255,0.010608000308275222
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,511,0.010627199709415436
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,511,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,1023,0.011664000153541566
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,1023,0.010527999699115753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,2047,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,2047,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,4095,0.014211200177669525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,4095,0.012508800625801087
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,8191,0.016465599834918975
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,float16,16383,0.030819201469421388
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,8191,0.014640000462532044
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,1,128,1,float16,fp8,16383,0.018486399948596955
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,1,0.008406399935483932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,1,0.008513599634170532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,3,0.008491200208663941
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,3,0.008399999886751174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,7,0.008553600311279297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,7,0.00852160006761551
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,15,0.008367999643087386
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,15,0.008403199911117553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,31,0.008528000116348267
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,31,0.008372800052165985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,63,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,63,0.008628799766302108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,127,0.008369600027799606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,127,0.009323199838399887
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,255,0.010236799716949463
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,255,0.00844319984316826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,511,0.008579199761152267
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,511,0.008460800349712371
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,1023,0.010232000052928925
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,1023,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,2047,0.010814400017261505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,2047,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,4095,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,8191,0.027110400795936584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,4095,0.013407999277114868
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,8191,0.014532800018787383
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,float16,16383,0.04529280066490173
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,2,128,1,float16,fp8,16383,0.0287200003862381
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,1,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,1,0.010028800368309021
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,3,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,3,0.009990400075912476
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,7,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,7,0.008497600257396699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,15,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,15,0.00857120007276535
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,31,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,31,0.008502399921417237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,63,0.010390400141477584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,63,0.008510400354862214
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,127,0.008444800227880477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,127,0.009550400078296661
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,255,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,255,0.00841279998421669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,511,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,511,0.008921600133180618
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,1023,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,1023,0.010360000282526016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,2047,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,2047,0.012720000743865967
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,4095,0.028479999303817748
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,4095,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,8191,0.04534879922866821
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,8191,0.028307199478149414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,float16,16383,0.07993119955062866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,1,0.008510400354862214
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,4,128,1,float16,fp8,16383,0.045126399397850035
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,1,0.008387199789285659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,3,0.008423999696969987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,3,0.008416000008583068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,7,0.008430399745702744
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,7,0.008497600257396699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,15,0.008422400057315826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,15,0.008343999832868576
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,31,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,31,0.00856959968805313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,63,0.008420799672603608
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,63,0.008374399691820144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,127,0.008430399745702744
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,127,0.00835679993033409
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,255,0.008483199775218964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,255,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,511,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,1023,0.012366399914026261
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,511,0.009536000341176987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,1023,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,2047,0.027039998769760133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,2047,0.012572799623012543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,4095,0.04469920098781586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,4095,0.027511999011039734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,8191,0.045228800177574156
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,8191,0.07849760055541992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,1,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,float16,16383,0.14822239875793458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,1,0.00846719965338707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,16,8,128,1,float16,fp8,16383,0.081768000125885
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,3,0.008660800009965896
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,3,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,7,0.010390400141477584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,7,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,15,0.01032159999012947
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,15,0.009273599833250046
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,31,0.008564800024032593
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,31,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,63,0.010339199751615524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,63,0.008417599648237229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,127,0.010275200009346008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,255,0.008473599702119828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,127,0.009892799705266953
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,255,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,511,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,511,0.010158400237560272
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,1023,0.010500799864530563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,2047,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,1023,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,2047,0.010558400303125381
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,4095,0.028137600421905516
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,4095,0.014496000111103058
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,8191,0.04524320065975189
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,8191,0.028891199827194215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,16383,0.07959359884262085
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,16383,0.04542079865932465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,1,0.00849440023303032
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,float16,32767,0.1472991943359375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,1,128,1,float16,fp8,32767,0.07996159791946411
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,1,0.008489599823951722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,3,0.008392000198364257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,3,0.008531200140714646
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,7,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,7,0.008339200168848038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,15,0.008382400125265121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,15,0.008500800281763077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,31,0.008387199789285659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,63,0.009217599779367447
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,31,0.008420799672603608
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,63,0.008422400057315826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,127,0.010232000052928925
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,127,0.008352000266313553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,255,0.009388799965381622
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,255,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,511,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,511,0.008460800349712371
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,1023,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,1023,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,2047,0.02693600058555603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,2047,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,4095,0.04490559995174408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,4095,0.02754879891872406
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,8191,0.07829920053482056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,8191,0.045291200280189514
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,16383,0.07953280210494995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,16383,0.1471935987472534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,fp8,32767,0.15136640071868895
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,2,128,1,float16,float16,32767,0.28297600746154783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,1,0.01058719977736473
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,1,0.009481599926948548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,3,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,7,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,3,0.01064639985561371
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,7,0.010494399815797806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,15,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,31,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,31,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,15,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,63,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,63,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,127,0.010812799632549285
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,127,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,255,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,511,0.012558400630950928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,511,0.010804799944162368
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,255,0.010545600205659866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,1023,0.026956799626350402
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,1023,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,2047,0.028240001201629637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,2047,0.04555999934673309
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,4095,0.07998560070991516
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,4095,0.04560160040855408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,8191,0.14928480386734008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,8191,0.08193600177764893
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,16383,0.2878159999847412
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,16383,0.15269440412521362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,1,0.010118400305509567
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,1,0.008816000074148178
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,float16,32767,0.5634399890899658
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,3,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,8,4,128,1,float16,fp8,32767,0.29418880939483644
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,7,0.010505600273609162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,3,0.008430399745702744
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,7,0.008531200140714646
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,15,0.01058719977736473
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,15,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,31,0.008540800213813782
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,63,0.008470399677753449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,31,0.009281600266695023
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,63,0.008457600325345992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,127,0.008462399989366532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,255,0.008907199651002885
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,127,0.009481599926948548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,255,0.008460800349712371
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,511,0.009398400038480758
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,511,0.010072000324726105
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,1023,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,1023,0.008432000130414962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,2047,0.009759999811649323
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,2047,0.010577599704265594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,4095,0.01130559965968132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,4095,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,8191,0.012624000012874604
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,8191,0.012598399817943574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,16383,0.015369600057601929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,16383,0.016803200542926788
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,float16,32767,0.01858240067958832
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,1,128,1,float16,fp8,32767,0.016524800658226015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,1,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,1,0.008524800091981888
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,3,0.008399999886751174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,7,0.00859680026769638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,3,0.008535999804735184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,15,0.008532799780368805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,7,0.008497600257396699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,15,0.008636800199747085
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,31,0.008537600189447403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,31,0.008486399799585343
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,63,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,63,0.008486399799585343
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,127,0.009467200189828873
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,127,0.01016160026192665
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,255,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,255,0.008430399745702744
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,511,0.010496000200510025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,1023,0.008457600325345992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,511,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,1023,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,2047,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,2047,0.010307200253009796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,4095,0.012368000298738479
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,4095,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,8191,0.012644800543785095
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,16383,0.01658879965543747
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,8191,0.012515200674533844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,float16,32767,0.018508799374103546
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,32767,0.018620799481868743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,2,128,1,float16,fp8,16383,0.016715200245380403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,1,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,1,0.008473599702119828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,3,0.010292799770832061
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,3,0.008408000320196151
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,7,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,7,0.008708799630403519
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,15,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,15,0.009412799775600434
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,31,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,31,0.008441600203514098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,63,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,63,0.009143999963998794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,127,0.008513599634170532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,127,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,255,0.010328000038862228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,255,0.00836160033941269
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,511,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,511,0.010171200335025787
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,1023,0.01037440001964569
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,1023,0.008606400340795517
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,2047,0.010592000186443329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,4095,0.01088479980826378
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,2047,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,4095,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,8191,0.014534400403499603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,8191,0.01273760050535202
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,16383,0.01656000018119812
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,float16,32767,0.01857919991016388
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,32767,0.01671680063009262
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,8,4,128,1,float16,fp8,16383,0.015699200332164764
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,1,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,1,0.009190399944782258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,3,0.008423999696969987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,3,0.008374399691820144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,7,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,15,0.009494400024414063
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,7,0.008481600135564805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,15,0.009256000071763993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,31,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,31,0.008419200032949447
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,63,0.009062399715185165
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,63,0.010267200320959092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,127,0.008673600107431411
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,127,0.008379200100898742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,255,0.010363200306892395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,255,0.00846880003809929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,511,0.008534400165081025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,1023,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,1023,0.008366400003433227
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,511,0.010334400087594986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,2047,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,2047,0.008587200194597244
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,4095,0.010574399679899215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,4095,0.010644800215959548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,8191,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,8191,0.012615999579429627
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,16383,0.016577599942684172
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,16383,0.016569599509239197
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,float16,32767,0.018751999735832213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,1,128,1,float16,fp8,32767,0.018532800674438476
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,1,0.009028799831867218
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,1,0.00865119993686676
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,3,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,3,0.008372800052165985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,7,0.010313600301742554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,7,0.008428800106048583
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,15,0.00950239971280098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,15,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,31,0.008504000306129456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,63,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,31,0.008462399989366532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,63,0.008451200276613235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,127,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,127,0.008502399921417237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,255,0.009017600119113922
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,511,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,255,0.010286399722099304
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,1023,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,1023,0.01034879982471466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,511,0.009806399792432785
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,2047,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,2047,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,4095,0.012443199753761292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,4095,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,8191,0.014470399916172027
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,8191,0.014561599493026734
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,16383,0.016364799439907075
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,16383,0.01465120017528534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,float16,32767,0.018651199340820313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,2,128,1,float16,fp8,32767,0.016595199704170227
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,1,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,3,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,1,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,3,0.010342399775981902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,7,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,15,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,7,0.01032480001449585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,15,0.010316800326108932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,31,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,31,0.008462399989366532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,63,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,63,0.00844319984316826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,127,0.010331200063228607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,127,0.008387199789285659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,255,0.010307200253009796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,255,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,511,0.01029760017991066
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,511,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,1023,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,1023,0.01034879982471466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,2047,0.010547199845314026
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,2047,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,4095,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,4095,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,8191,0.014398400485515595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,16383,0.01653759926557541
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,8191,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,16383,0.014564800262451171
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,float16,32767,0.03099200129508972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,8,4,128,1,float16,fp8,32767,0.016572800278663636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,1,0.008364800363779068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,1,0.008449599891901017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,3,0.008388800173997879
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,3,0.008499199897050858
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,7,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,7,0.008470399677753449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,15,0.00835840031504631
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,31,0.008406399935483932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,15,0.008475200086832047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,31,0.008377599716186523
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,63,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,63,0.00840959995985031
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,127,0.010331200063228607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,127,0.00846560001373291
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,255,0.010300800204277039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,255,0.008454400300979614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,511,0.010044799745082855
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,511,0.008403199911117553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,1023,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,1023,0.008473599702119828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,2047,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,2047,0.010865599662065507
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,4095,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,4095,0.010835199803113937
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,8191,0.014473600685596466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,8191,0.027419200539588927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,16383,0.04519200026988983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,16383,0.02752000093460083
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,1,0.010360000282526016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,fp8,32767,0.04552319943904877
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,1,128,1,float16,float16,32767,0.079830402135849
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,3,0.010304000228643417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,1,0.00840959995985031
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,3,0.008723200112581254
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,7,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,7,0.008504000306129456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,15,0.010328000038862228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,15,0.009867200255393982
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,31,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,63,0.01024480015039444
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,31,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,63,0.008667200058698653
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,127,0.010299199819564819
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,127,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,255,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,255,0.00873439982533455
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,511,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,511,0.009982399642467499
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,1023,0.01053439974784851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,2047,0.012515200674533844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,2047,0.01223199963569641
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,1023,0.010315199941396713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,4095,0.026932799816131593
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,4095,0.014478400349617004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,8191,0.04521600008010864
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,8191,0.02741599977016449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,16383,0.07958239912986756
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,16383,0.04552960097789764
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,1,0.008422400057315826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,float16,32767,0.14726879596710205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,1,0.008504000306129456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,2,128,1,float16,fp8,32767,0.0800607979297638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,3,0.00838399976491928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,3,0.008470399677753449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,7,0.008558399975299835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,7,0.008406399935483932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,15,0.008507200330495835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,15,0.008449599891901017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,31,0.008483199775218964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,31,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,63,0.008532799780368805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,63,0.008395200222730636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,127,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,127,0.008515200018882752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,255,0.009335999935865402
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,255,0.00846719965338707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,511,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,511,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,1023,0.012412799894809723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,1023,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,2047,0.027351999282836915
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,2047,0.013544000685214996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,4095,0.027878400683403016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,4095,0.04522559940814972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,8191,0.04538399875164032
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,8191,0.07996000051498413
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,16383,0.07986720204353333
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,16383,0.14797120094299315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,1,0.010345599800348281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,1,0.008444800227880477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,float16,32767,0.28297760486602785
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,3,0.00862400010228157
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,8,4,128,1,float16,fp8,32767,0.14847359657287598
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,7,0.009911999851465226
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,3,0.008630400151014328
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,7,0.008585599809885025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,15,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,31,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,31,0.010335999727249145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,15,0.008633600175380706
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,63,0.010384000092744827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,63,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,127,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,127,0.008478400111198426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,255,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,511,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,255,0.008659200370311737
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,511,0.010284800082445145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,1023,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,1023,0.010543999820947647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,2047,0.010553599894046783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,2047,0.010475199669599533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,4095,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,4095,0.012585599720478059
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,8191,0.014444799721240997
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,8191,0.014494399726390838
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,16383,0.016758400201797485
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,16383,0.014804799854755402
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,1,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,float16,32767,0.01881919950246811
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,1,128,1,float16,fp8,32767,0.01683039963245392
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,1,0.00888959988951683
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,3,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,3,0.00862400010228157
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,7,0.010311999917030334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,7,0.008662399649620057
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,15,0.010526400059461594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,15,0.00873280018568039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,31,0.01029599979519844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,63,0.010294400155544281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,31,0.008497600257396699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,63,0.009563200175762177
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,127,0.008449599891901017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,127,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,255,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,255,0.00835999995470047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,511,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,511,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,1023,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,1023,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,2047,0.010531199723482132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,2047,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,4095,0.012572799623012543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,8191,0.014496000111103058
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,4095,0.012366399914026261
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,16383,0.01655520051717758
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,8191,0.012571200728416443
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,16383,0.014582400023937226
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,float16,32767,0.031041601300239564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,2,128,1,float16,fp8,32767,0.01646080017089844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,1,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,1,0.008585599809885025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,3,0.010371199995279311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,3,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,7,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,7,0.01035040020942688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,15,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,15,0.01034879982471466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,31,0.010382399708032609
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,31,0.008479999750852585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,63,0.010315199941396713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,63,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,127,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,127,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,255,0.010335999727249145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,255,0.009401600062847137
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,511,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,511,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,1023,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,1023,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,2047,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,2047,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,4095,0.012670400738716125
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,4095,0.01242239996790886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,8191,0.014657600224018097
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,8191,0.012585599720478059
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,16383,0.03028320074081421
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,16383,0.0165120005607605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,fp8,32767,0.030011200904846193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,8,4,128,1,float16,float16,32767,0.04736000001430511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,1,0.009030400216579438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,1,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,3,0.009294400364160538
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,3,0.008495999872684479
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,7,0.008932799845933915
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,7,0.008504000306129456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,15,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,15,0.00931520015001297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,31,0.008539199829101562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,31,0.008417599648237229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,63,0.008430399745702744
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,63,0.008448000252246856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,127,0.008481600135564805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,127,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,255,0.009180799871683121
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,255,0.008483199775218964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,511,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,1023,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,1023,0.010515200346708298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,511,0.010625600069761276
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,2047,0.0271807998418808
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,2047,0.012566399574279786
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,4095,0.0443343997001648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,4095,0.028563201427459717
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,8191,0.0790880024433136
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,8191,0.045132800936698914
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,1,0.010384000092744827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,float16,16383,0.14773759841918946
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,1,128,1,float16,fp8,16383,0.0796992003917694
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,1,0.009401600062847137
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,3,0.01032480001449585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,3,0.010540799796581268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,7,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,7,0.010339199751615524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,15,0.010569600015878677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,15,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,31,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,31,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,63,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,63,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,127,0.010552000254392624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,127,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,255,0.01050880029797554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,255,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,511,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,511,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,1023,0.014550399780273438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,1023,0.02728320062160492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,2047,0.04516800045967102
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,2047,0.02770879864692688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,4095,0.07993280291557311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,4095,0.04677119851112366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,8191,0.1478816032409668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,8191,0.08177279829978942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,1,0.012583999335765839
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,float16,16383,0.2861727952957153
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,1,0.01255200058221817
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,2,128,1,float16,fp8,16383,0.152401602268219
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,3,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,3,0.012624000012874604
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,7,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,15,0.012622399628162384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,7,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,31,0.01255359947681427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,15,0.012385600060224534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,31,0.012433599680662155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,63,0.012535999715328216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,127,0.012542399764060973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,63,0.012455999851226807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,127,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,255,0.012435200065374375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,255,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,511,0.02815360128879547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,511,0.014542399346828461
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,1023,0.046438398957252505
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,1023,0.02890399992465973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,2047,0.08101279735565185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,2047,0.047435200214385985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,4095,0.15183199644088746
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,4095,0.08228160142898559
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,8191,0.2944256067276001
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,8191,0.1520848035812378
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,1,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,1,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,3,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,float16,16383,0.5797728061676025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,3,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,7,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,8,4,128,1,float16,fp8,16383,0.2958751916885376
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,15,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,7,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,15,0.010279999673366546
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,31,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,31,0.010700800269842149
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,63,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,63,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,127,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,127,0.010681600123643876
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,255,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,255,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,511,0.011044800281524658
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,1023,0.02704800069332123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,511,0.012772800028324127
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,1023,0.014451199769973755
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,2047,0.04526880085468292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,2047,0.028988799452781676
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,float16,4095,0.07992159724235534
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,1,128,1,float16,fp8,4095,0.04575360119342804
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,1,0.012720000743865967
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,1,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,3,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,3,0.01265760064125061
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,7,0.012606400251388549
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,7,0.012462399899959564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,15,0.012404800206422806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,31,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,15,0.012675200402736665
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,63,0.01268479973077774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,31,0.012510399520397186
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,63,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,127,0.01250080019235611
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,127,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,255,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,255,0.012452799826860428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,511,0.01456640064716339
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,511,0.027990400791168213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,1023,0.04588640034198761
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,1023,0.028891199827194215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,2047,0.04704799950122833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,2047,0.08125920295715332
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,1,0.016769599914550782
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,float16,4095,0.15170719623565673
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,2,128,1,float16,fp8,4095,0.0820367991924286
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,1,0.01467359960079193
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,7,0.016675199568271636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,3,0.015438400208950043
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,3,0.01661919951438904
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,7,0.014644800126552582
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,15,0.016663999855518342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,15,0.014555199444293976
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,31,0.01664479970932007
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,31,0.016470399498939515
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,63,0.016633599996566772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,127,0.016700799763202667
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,63,0.016543999314308167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,127,0.014571200311183929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,255,0.026851201057434083
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,255,0.01648000031709671
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,511,0.04536640048027039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,511,0.02853600084781647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,1023,0.04687039852142334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,1023,0.08015199899673461
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,2047,0.14786880016326903
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,2047,0.08146880269050598
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,1,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,float16,4095,0.28631999492645266
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,1,0.012582400441169738
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,8,4,128,1,float16,fp8,4095,0.15160640478134155
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,3,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,3,0.012558400630950928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,7,0.012464000284671784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,7,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,15,0.012656000256538392
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,15,0.012489599734544754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,31,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,31,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,63,0.01244639977812767
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,63,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,127,0.012590399384498597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,127,0.012540799379348756
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,255,0.012510399520397186
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,255,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,511,0.028788799047470094
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,511,0.014502400159835815
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,1023,0.04613119959831238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,1023,0.029019200801849367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,1,0.01661760061979294
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,float16,2047,0.08182240128517151
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,1,128,1,float16,fp8,2047,0.04747680127620697
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,1,0.01645440012216568
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,3,0.016633599996566772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,3,0.016414399445056915
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,7,0.016569599509239197
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,7,0.016553600132465363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,15,0.01669439971446991
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,15,0.016598400473594666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,31,0.017603200674057008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,31,0.01643519997596741
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,63,0.016782400012016297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,127,0.016729600727558136
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,63,0.016568000614643096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,127,0.01542080044746399
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,255,0.0270687997341156
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,255,0.01493919938802719
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,511,0.04528000056743622
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,511,0.028830400109291075
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,1023,0.08005279898643494
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,1023,0.04708479940891266
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,fp8,2047,0.08187199831008911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,2,128,1,float16,float16,2047,0.14805760383605956
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,1,0.026862400770187377
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,1,0.022873599827289582
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,3,0.026790401339530943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,3,0.02290239930152893
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,7,0.026791998744010927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,7,0.022814400494098663
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,15,0.026795199513435362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,15,0.022856000065803527
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,31,0.026862400770187377
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,31,0.022835199534893037
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,63,0.02680320143699646
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,127,0.03089759945869446
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,63,0.0230320006608963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,127,0.02281759977340698
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,255,0.045737600326538085
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,255,0.029455998539924623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,511,0.08032960295677186
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,511,0.04742560088634491
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,1023,0.0819167971611023
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,1023,0.14934240579605101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,float16,2047,0.28689119815826414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,1,0.010523200035095215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,8,4,128,1,float16,fp8,2047,0.15080640316009522
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,3,0.010556799918413162
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,1,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,7,0.010521599650382995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,7,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,3,0.008606400340795517
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,31,0.009105599671602248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,15,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,15,0.010443200170993806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,31,0.00843999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,63,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,63,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,127,0.009603200107812881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,127,0.009524799883365631
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,255,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,511,0.010337600111961364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,255,0.00843520015478134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,1023,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,1023,0.010337600111961364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,511,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,2047,0.010601600259542465
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,2047,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,4095,0.01252480000257492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,4095,0.012337599694728852
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,8191,0.014571200311183929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,16383,0.016451199352741242
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,8191,0.012699200212955475
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,16383,0.014558400213718414
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,float16,32767,0.030828800797462464
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,1,0.009716799855232239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,1,0.00923840031027794
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,3,0.00952799990773201
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,1,128,1,float16,fp8,32767,0.016684800386428833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,3,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,7,0.009620799869298934
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,7,0.010062400251626968
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,15,0.010328000038862228
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,15,0.008488000184297562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,31,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,31,0.010308799892663955
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,63,0.009427200257778167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,127,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,63,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,127,0.008398400247097015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,255,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,255,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,511,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,511,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,1023,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,1023,0.010460799932479859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,2047,0.011107199639081956
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,2047,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,4095,0.012428800016641617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,4095,0.012585599720478059
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,8191,0.014584000408649444
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,8191,0.012590399384498597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,16383,0.030561599135398864
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,16383,0.016497600078582763
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,float16,32767,0.04737280011177063
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,2,128,1,float16,fp8,32767,0.031004801392555237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,1,0.009043200314044953
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,1,0.008406399935483932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,3,0.008449599891901017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,3,0.009225600212812424
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,7,0.008532799780368805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,7,0.008376000076532364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,15,0.009468799829483033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,15,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,31,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,31,0.008528000116348267
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,63,0.00950239971280098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,63,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,127,0.010276799649000167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,127,0.008452799916267396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,255,0.009487999975681305
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,255,0.008369600027799606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,511,0.008486399799585343
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,511,0.008398400247097015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,1023,0.01027199998497963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,2047,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,1023,0.00838399976491928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,4095,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,4095,0.01252480000257492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,2047,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,8191,0.02696160078048706
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,8191,0.014535999298095703
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,16383,0.04535360038280487
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,16383,0.02728320062160492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,float16,32767,0.079585599899292
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,1,0.016638399660587312
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,8,4,128,1,float16,fp8,32767,0.04543839991092682
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,1,0.016510400176048278
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,3,0.01669919937849045
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,7,0.01669919937849045
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,7,0.01664479970932007
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,3,0.01669600009918213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,15,0.018488000333309173
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,15,0.016542400419712066
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,31,0.018318399786949158
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,31,0.01674239933490753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,63,0.018518400192260743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,127,0.018544000387191773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,63,0.016715200245380403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,127,0.01656160056591034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,255,0.02887679934501648
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,511,0.04564320147037506
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,255,0.01671680063009262
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,511,0.02919679880142212
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,float16,1023,0.08024640083312988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,1,0.026899200677871705
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,1,128,1,float16,fp8,1023,0.047723200917243955
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,1,0.024105599522590636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,3,0.026910400390625
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,3,0.02455199956893921
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,7,0.026841598749160766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,7,0.024823999404907225
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,15,0.026862400770187377
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,31,0.026848000288009644
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,15,0.02470560073852539
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,31,0.02468000054359436
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,63,0.026867198944091796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,63,0.024702399969100952
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,127,0.024659200012683867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,127,0.030419200658798218
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,255,0.04546720087528229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,255,0.030395200848579405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,511,0.08026559948921204
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,511,0.047393599152565004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,1,0.045187199115753175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,float16,1023,0.1497648000717163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,2,128,1,float16,fp8,1023,0.08270400166511535
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,1,0.03909760117530823
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,3,0.04531840085983276
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,7,0.04524959921836853
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,3,0.039238399267196654
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,7,0.039017599821090695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,15,0.04525440037250519
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,15,0.03917759954929352
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,31,0.04520800113677979
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,31,0.039134401082992556
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,63,0.04748159945011139
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,63,0.03916319906711578
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,127,0.04984959959983826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,127,0.04126560091972351
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,255,0.04875679910182953
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,255,0.0818127989768982
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,511,0.14990400075912474
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,511,0.0824176013469696
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,1,0.026840001344680786
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,float16,1023,0.28728160858154295
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,1,0.024732799828052522
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,8,4,128,1,float16,fp8,1023,0.1525823950767517
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,3,0.024798400700092316
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,3,0.026927998661994933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,7,0.026926401257514953
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,7,0.024804799258708952
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,15,0.027003198862075806
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,15,0.024905599653720856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,31,0.024831999838352204
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,31,0.026953598856925963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,63,0.026878398656845093
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,63,0.02484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,127,0.024723200500011443
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,127,0.030870398879051207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,255,0.03102400004863739
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,255,0.047224000096321106
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,float16,511,0.08211359977722169
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,1,0.04532159864902496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,1,128,1,float16,fp8,511,0.04870879948139191
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,1,0.04114879965782166
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,3,0.04531520009040833
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,3,0.03928160071372986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,7,0.04528000056743622
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,7,0.03917919993400574
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,15,0.045363199710845944
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,15,0.03920319974422455
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,31,0.0453904002904892
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,31,0.039084801077842714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,63,0.03911519944667816
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,63,0.047419199347496034
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,127,0.05159839987754822
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,127,0.04317440092563629
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,255,0.08224959969520569
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,255,0.05088480114936829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,float16,511,0.15107359886169433
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,2,128,1,float16,fp8,511,0.08440960049629212
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,1,0.08221120238304139
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,1,0.06992160081863404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,3,0.08216320276260376
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,3,0.06990240216255188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,7,0.06988959908485412
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,7,0.08212320208549499
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,15,0.08391039967536926
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,15,0.06992160081863404
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,31,0.0841152012348175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,31,0.07196800112724304
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,63,0.08412320017814637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,63,0.07201120257377625
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,127,0.09051520228385926
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,127,0.07436320185661316
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,255,0.1519327998161316
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,255,0.08948320150375366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,1,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,1,0.010339199751615524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,fp8,511,0.15465760231018066
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,8,4,128,1,float16,float16,511,0.28928959369659424
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,3,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,3,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,7,0.010331200063228607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,7,0.010318399965763092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,15,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,31,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,15,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,31,0.010344000160694122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,63,0.010313600301742554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,63,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,127,0.010497599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,127,0.010326399654150008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,255,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,511,0.010384000092744827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,511,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,255,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,1023,0.010391999781131745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,2047,0.010664000362157821
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,1023,0.010361599922180175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,4095,0.012545600533485413
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,2047,0.010526400059461594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,4095,0.012428800016641617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,8191,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,8191,0.01388159990310669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,16383,0.02953439950942993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,16383,0.01583999991416931
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,float16,32767,0.04738720059394837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,1,0.0083856001496315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,1,128,1,float16,fp8,32767,0.030801600217819212
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,1,0.008516799658536911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,3,0.008406399935483932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,3,0.008532799780368805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,7,0.008430399745702744
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,7,0.008392000198364257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,15,0.009388799965381622
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,31,0.00835999995470047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,15,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,31,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,63,0.00846880003809929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,63,0.008462399989366532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,127,0.009465599805116654
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,127,0.008372800052165985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,255,0.00849440023303032
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,255,0.008489599823951722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,511,0.008380799740552902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,511,0.008774399757385254
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,1023,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,1023,0.008350399881601333
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,2047,0.010948800295591355
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,4095,0.012542399764060973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,2047,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,4095,0.01252640038728714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,8191,0.026927998661994933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,16383,0.04532960057258606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,8191,0.014406399428844452
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,16383,0.027371200919151305
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,float16,32767,0.07826560139656066
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,2,128,1,float16,fp8,32767,0.045326399803161624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,1,0.008425600081682205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,3,0.00843520015478134
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,1,0.00846719965338707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,3,0.0095040000975132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,7,0.008428800106048583
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,15,0.010000000149011612
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,7,0.008470399677753449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,15,0.008460800349712371
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,31,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,31,0.008895999938249587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,63,0.010040000081062317
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,63,0.00862400010228157
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,127,0.010371199995279311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,127,0.008603200316429138
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,255,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,255,0.008579199761152267
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,511,0.01029599979519844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,511,0.008406399935483932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,1023,0.010531199723482132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,1023,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,2047,0.012665599584579468
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,2047,0.010529600083827972
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,4095,0.028193598985671996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,4095,0.013116799294948578
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,8191,0.027958399057388304
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,8191,0.045284798741340636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,16383,0.045414400100708005
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,16383,0.08049439787864685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,float16,32767,0.14820799827575684
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,1,0.01053439974784851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,8,4,128,1,float16,fp8,32767,0.08030880093574524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,1,0.010300800204277039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,3,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,3,0.008515200018882752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,7,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,7,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,15,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,15,0.010503999888896942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,31,0.008428800106048583
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,31,0.0094991996884346
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,63,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,63,0.008416000008583068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,127,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,255,0.008473599702119828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,127,0.010316800326108932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,255,0.009459199756383896
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,511,0.009926400333642959
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,511,0.010579200088977813
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,1023,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,1023,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,2047,0.012705600261688233
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,4095,0.026931199431419372
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,2047,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,8191,0.04523360133171082
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,4095,0.014686399698257446
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,8191,0.02784799933433533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,16383,0.07889440059661865
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,16383,0.04556959867477417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,32767,0.14768320322036743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,32767,0.07981280088424683
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,1,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,1,0.008392000198364257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,fp8,65535,0.14816160202026368
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,1,128,1,float16,float16,65535,0.28392319679260253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,3,0.008495999872684479
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,7,0.008369600027799606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,3,0.008427199721336365
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,15,0.00844319984316826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,7,0.008662399649620057
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,15,0.00846880003809929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,31,0.00851840004324913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,31,0.008403199911117553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,63,0.009428799897432328
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,63,0.008432000130414962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,127,0.008607999980449676
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,127,0.008531200140714646
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,255,0.008414400368928909
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,511,0.010487999767065048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,255,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,511,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,1023,0.012409599870443344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,1023,0.010590399801731109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,2047,0.0268095999956131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,2047,0.012692800164222718
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,4095,0.0280239999294281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,4095,0.04421919882297516
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,8191,0.07820000052452088
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,8191,0.045459198951721194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,16383,0.146070396900177
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,16383,0.07995520234107971
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,32767,0.2834255933761597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,32767,0.14785759449005126
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,1,0.008703999966382981
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,1,0.008420799672603608
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,3,0.009147199988365173
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,float16,65535,0.5556367874145508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,3,0.00841120034456253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,7,0.008484800159931184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,4,2,128,1,float16,fp8,65535,0.2886719942092896
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,15,0.008817599713802337
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,7,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,31,0.009591999650001525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,31,0.009355200082063675
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,15,0.008504000306129456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,127,0.008441600203514098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,63,0.009622400254011154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,63,0.008499199897050858
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,127,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,255,0.008436799794435502
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,511,0.00843999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,511,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,255,0.00862559974193573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,1023,0.010390400141477584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,1023,0.008483199775218964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,2047,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,4095,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,2047,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,4095,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,8191,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,8191,0.01271360069513321
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,16383,0.016760000586509706
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,16383,0.014539200067520141
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,32767,0.01661760061979294
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,32767,0.018748800456523895
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,float16,65535,0.01865919977426529
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,1,0.008430399745702744
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,1,128,1,float16,fp8,65535,0.018587200343608855
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,1,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,3,0.008374399691820144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,3,0.010572800040245056
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,7,0.010326399654150008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,7,0.008380799740552902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,15,0.010214400291442872
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,15,0.00854720026254654
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,31,0.008340799808502197
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,31,0.008481600135564805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,63,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,63,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,127,0.00840959995985031
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,127,0.008588799834251403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,255,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,255,0.008479999750852585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,511,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,511,0.008495999872684479
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,1023,0.008392000198364257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,2047,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,1023,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,2047,0.010284800082445145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,4095,0.011209599673748016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,8191,0.012567999958992004
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,4095,0.010521599650382995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,8191,0.012503999471664428
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,16383,0.01855680048465729
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,16383,0.01669600009918213
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,32767,0.018580800294876097
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,32767,0.018620799481868743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,fp8,65535,0.01870879977941513
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,4,2,128,1,float16,float16,65535,0.02075359970331192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,1,0.008387199789285659
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,1,0.008449599891901017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,3,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,3,0.008428800106048583
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,7,0.010286399722099304
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,7,0.008553600311279297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,15,0.008478400111198426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,15,0.008472000062465668
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,31,0.008395200222730636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,31,0.010371199995279311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,63,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,127,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,63,0.008398400247097015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,127,0.008478400111198426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,255,0.008379200100898742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,255,0.00852160006761551
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,511,0.00844319984316826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,511,0.008856000006198883
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,1023,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,1023,0.00840959995985031
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,2047,0.01051200032234192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,2047,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,4095,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,4095,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,8191,0.012732799351215362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,8191,0.012540799379348756
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,16383,0.017694400250911714
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,16383,0.016705599427223206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,32767,0.018590399622917177
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,32767,0.018595199286937713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,1,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,float16,65535,0.020707200467586517
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,1,128,1,float16,fp8,65535,0.018619200587272643
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,1,0.008388800173997879
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,3,0.009681600332260131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,3,0.008449599891901017
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,7,0.00941760018467903
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,15,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,7,0.008508799970149994
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,15,0.008454400300979614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,31,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,31,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,63,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,127,0.010339199751615524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,127,0.008515200018882752
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,63,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,255,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,255,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,511,0.010342399775981902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,511,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,1023,0.010390400141477584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,1023,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,2047,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,2047,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,4095,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,4095,0.012430399656295776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,8191,0.014572800695896148
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,8191,0.014452800154685974
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,16383,0.016625599563121797
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,16383,0.01647839993238449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,32767,0.01865759938955307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,float16,65535,0.03421280086040497
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,32767,0.016731199622154237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,1,0.00852160006761551
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,4,2,128,1,float16,fp8,65535,0.01860000044107437
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,3,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,1,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,3,0.008353599905967712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,7,0.008462399989366532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,7,0.008408000320196151
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,15,0.008448000252246856
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,15,0.00841120034456253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,31,0.008486399799585343
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,31,0.008379200100898742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,63,0.008479999750852585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,63,0.008508799970149994
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,127,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,127,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,255,0.008577600121498108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,255,0.008423999696969987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,511,0.008660800009965896
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,511,0.008374399691820144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,1023,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,1023,0.010078399628400802
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,2047,0.01058880016207695
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,2047,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,4095,0.011774399876594543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,4095,0.012620800733566284
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,8191,0.027764800190925597
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,8191,0.014417600631713868
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,16383,0.04508799910545349
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,16383,0.02812960147857666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,32767,0.0786575973033905
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,32767,0.04546880125999451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,fp8,65535,0.08017600178718567
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,1,0.010577599704265594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,1,128,1,float16,float16,65535,0.14683519601821898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,1,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,3,0.010785599797964096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,3,0.00841120034456253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,7,0.010363200306892395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,15,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,7,0.010608000308275222
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,15,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,31,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,63,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,63,0.00952960029244423
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,31,0.009440000355243682
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,127,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,127,0.009803199768066406
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,255,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,255,0.010361599922180175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,511,0.009926400333642959
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,1023,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,1023,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,511,0.009540800005197525
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,2047,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,2047,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,4095,0.027535998821258546
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,8191,0.044784000515937804
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,4095,0.014446400105953217
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,8191,0.028340798616409302
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,16383,0.04556320011615753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,16383,0.07822560071945191
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,32767,0.14783519506454468
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,32767,0.08056960105895997
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,1,0.009932799637317658
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,1,0.008459199965000153
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,float16,65535,0.2843983888626099
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,4,2,128,1,float16,fp8,65535,0.14919999837875367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,3,0.01029760017991066
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,3,0.008367999643087386
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,7,0.00849440023303032
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,7,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,15,0.008406399935483932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,15,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,31,0.009515199810266495
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,63,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,31,0.010452800244092942
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,63,0.008488000184297562
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,127,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,127,0.010566399991512298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,255,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,255,0.009857600182294845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,511,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,511,0.009723199903964997
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,1023,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,2047,0.01037440001964569
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,1023,0.010451199859380722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,2047,0.01032319962978363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,4095,0.012489599734544754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,4095,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,8191,0.014609600603580474
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,8191,0.014473600685596466
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,16383,0.015908800065517426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,16383,0.014963200688362122
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,32767,0.01871200054883957
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,32767,0.016598400473594666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,float16,65535,0.034615999460220336
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,1,128,1,float16,fp8,65535,0.018859200179576874
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,1,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,1,0.009915199875831605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,3,0.01032319962978363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,7,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,3,0.008457600325345992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,7,0.009700799733400345
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,15,0.010366400331258773
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,15,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,31,0.010390400141477584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,31,0.010289599746465683
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,63,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,63,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,127,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,127,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,255,0.010276799649000167
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,255,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,511,0.010552000254392624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,1023,0.010523200035095215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,511,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,2047,0.010515200346708298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,2047,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,1023,0.010543999820947647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,4095,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,4095,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,8191,0.014616000652313232
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,8191,0.012630400061607362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,16383,0.016524800658226015
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,32767,0.031046399474143983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,16383,0.014528000354766845
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,32767,0.0166143998503685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,1,0.008475200086832047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,float16,65535,0.048763200640678406
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,3,0.008497600257396699
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,1,0.008478400111198426
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,4,2,128,1,float16,fp8,65535,0.031404799222946166
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,3,0.008491200208663941
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,7,0.008484800159931184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,7,0.008516799658536911
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,15,0.00849440023303032
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,15,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,31,0.008510400354862214
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,31,0.008428800106048583
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,63,0.008982399851083756
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,127,0.009838400036096573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,63,0.00857279971241951
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,127,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,255,0.009014400094747544
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,255,0.008558399975299835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,511,0.011110399663448334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,511,0.010339199751615524
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,1023,0.012574400007724761
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,1023,0.010502400249242783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,2047,0.027062401175498962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,2047,0.012647999823093415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,4095,0.027710399031639098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,4095,0.04540160000324249
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,8191,0.045259198546409606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,8191,0.07867199778556824
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,16383,0.14766240119934082
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,16383,0.07953280210494995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,1,0.010371199995279311
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,fp8,32767,0.14803040027618408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,1,128,1,float16,float16,32767,0.2840847969055176
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,1,0.010313600301742554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,7,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,3,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,7,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,3,0.01053600013256073
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,15,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,15,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,31,0.010566399991512298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,31,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,63,0.010471999645233154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,63,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,127,0.010523200035095215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,127,0.010360000282526016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,255,0.0105103999376297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,511,0.012601600587368011
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,255,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,511,0.010553599894046783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,1023,0.02735520005226135
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,1023,0.014459200203418732
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,2047,0.04494880139827728
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,2047,0.02898080050945282
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,4095,0.07986559867858886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,4095,0.04525440037250519
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,8191,0.14831520318984986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,8191,0.08170719742774964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,16383,0.2868000030517578
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,16383,0.15362880229949952
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,fp8,32767,0.29625439643859863
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,4,2,128,1,float16,float16,32767,0.5650415897369385
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,1,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,3,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,3,0.010361599922180175
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,1,0.010422399640083313
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,7,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,15,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,7,0.010628800094127654
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,31,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,15,0.010363200306892395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,63,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,31,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,63,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,127,0.010473600029945374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,255,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,255,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,127,0.010567999631166457
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,511,0.012638400495052337
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,511,0.011631999909877778
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,1023,0.01446239948272705
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,1023,0.02731359899044037
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,2047,0.04532960057258606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,2047,0.028140801191329955
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,4095,0.04605759978294373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,4095,0.07965440154075623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,float16,8191,0.14848799705505372
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,1,0.012668800354003907
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,1,128,1,float16,fp8,8191,0.08022080063819885
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,1,0.012390399724245072
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,3,0.012668800354003907
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,7,0.012537600100040435
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,3,0.012561599910259246
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,7,0.012624000012874604
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,15,0.012561599910259246
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,15,0.012537600100040435
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,31,0.012399999797344208
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,31,0.012455999851226807
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,63,0.012516799569129943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,63,0.012412799894809723
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,127,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,127,0.012491200119256973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,255,0.012639999389648438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,255,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,511,0.014660799503326416
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,511,0.02754879891872406
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,1023,0.029016000032424927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,1023,0.04547199904918671
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,2047,0.08093439936637878
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,2047,0.047336000204086306
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,4095,0.15095839500427247
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,4095,0.08192960023880005
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,1,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,1,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,3,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,float16,8191,0.29223198890686036
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,4,2,128,1,float16,fp8,8191,0.15132639408111573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,3,0.012491200119256973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,15,0.01255040019750595
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,7,0.012539200484752655
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,7,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,15,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,31,0.012641599774360657
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,31,0.01247519999742508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,63,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,63,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,127,0.012491200119256973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,127,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,255,0.012678399682044983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,255,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,511,0.01464959979057312
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,511,0.02850080132484436
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,1023,0.04601759910583496
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,1023,0.029039999842643736
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,2047,0.08157600164413452
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,2047,0.04701600074768066
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,1,0.016595199704170227
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,float16,4095,0.15183199644088746
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,1,128,1,float16,fp8,4095,0.08223999738693237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,1,0.014628799259662628
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,3,0.016657599806785585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,7,0.016579200327396394
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,3,0.016550399363040924
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,7,0.016443200409412384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,15,0.016655999422073364
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,15,0.016545599699020384
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,31,0.018343999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,31,0.015532800555229187
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,63,0.016739200055599212
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,63,0.01650879979133606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,127,0.016598400473594666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,127,0.016572800278663636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,255,0.026940798759460448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,255,0.015510399639606477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,511,0.04508320093154907
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,511,0.028787198662757873
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,1023,0.07934560179710388
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,1023,0.04688799977302551
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,2047,0.14759839773178102
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,2047,0.0812831997871399
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,1,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,float16,4095,0.2848128080368042
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,1,0.010299199819564819
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,3,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,4,2,128,1,float16,fp8,4095,0.15014879703521727
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,3,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,7,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,15,0.00843999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,7,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,15,0.008876799792051315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,31,0.008555199950933456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,31,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,63,0.010315199941396713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,63,0.00843840017914772
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,127,0.010342399775981902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,127,0.009593600034713745
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,255,0.008774399757385254
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,255,0.010313600301742554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,511,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,511,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,1023,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,1023,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,2047,0.010411199927330018
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,2047,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,4095,0.012398400157690049
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,4095,0.012521600723266602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,8191,0.014494399726390838
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,8191,0.012972800433635712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,16383,0.016740800440311433
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,16383,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,32767,0.030827200412750243
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,32767,0.01661760061979294
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,float16,65535,0.04911839962005615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,1,128,1,float16,fp8,65535,0.0310591995716095
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,1,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,3,0.010927999764680863
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,1,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,3,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,7,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,7,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,15,0.010364799946546554
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,15,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,31,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,31,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,63,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,63,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,127,0.010388799756765366
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,127,0.0103472001850605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,255,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,255,0.010340800136327743
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,511,0.01043360009789467
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,511,0.010356800258159637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,1023,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,1023,0.01045600026845932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,2047,0.010518400371074677
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,2047,0.01045759990811348
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,4095,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,4095,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,8191,0.012939199805259705
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,8191,0.014726400375366211
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,16383,0.02956160008907318
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,16383,0.01650879979133606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,32767,0.04760800004005432
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,32767,0.030939200520515443
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,float16,65535,0.08109120130538941
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,1,0.017975999414920805
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,4,2,128,1,float16,fp8,65535,0.04778079986572266
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,1,0.016603200137615202
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,3,0.01722240000963211
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,3,0.016582399606704712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,7,0.01658399999141693
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,7,0.01655679941177368
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,15,0.016734400391578676
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,15,0.016598400473594666
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,31,0.018555200099945067
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,63,0.016667200624942778
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,31,0.016459199786186218
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,63,0.016652800142765045
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,127,0.01659200042486191
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,255,0.027651199698448183
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,127,0.01663520038127899
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,255,0.016551999747753142
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,511,0.04544160068035126
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,511,0.028971201181411742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,1023,0.08013759851455689
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,1023,0.047177600860595706
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,1,0.026788800954818726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,float16,2047,0.1479632019996643
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,1,128,1,float16,fp8,2047,0.08231840133666993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,1,0.02470560073852539
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,3,0.02486560046672821
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,3,0.026923200488090514
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,7,0.026929599046707154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,7,0.0247871994972229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,15,0.026878398656845093
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,15,0.024769599735736846
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,31,0.026774400472640993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,63,0.026892799139022826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,31,0.02462079972028732
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,63,0.0247871994972229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,127,0.030723199248313904
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,127,0.02468159943819046
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,255,0.04543519914150238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,255,0.030513599514961243
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,511,0.04732159972190857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,511,0.08026559948921204
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,1023,0.08271999955177307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,1023,0.14804480075836182
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,fp8,2047,0.1513424038887024
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,4,2,128,1,float16,float16,2047,0.2845344066619873
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,1,0.026791998744010927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,1,0.024926400184631346
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,3,0.026843199133872987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,3,0.024864000082015992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,7,0.026848000288009644
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,15,0.02686080038547516
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,15,0.024777600169181825
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,7,0.024967999756336214
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,31,0.02693760097026825
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,31,0.024873599410057068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,63,0.027046400308609008
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,127,0.030947199463844298
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,63,0.0248416006565094
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,127,0.025062400102615356
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,255,0.04561119973659515
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,255,0.031225600838661195
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,511,0.04791519939899445
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,511,0.08047360181808472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,float16,1023,0.14962719678878783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,1,0.045368000864982605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,1,128,1,float16,fp8,1023,0.08361120223999023
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,1,0.04126720130443573
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,3,0.04551520049571991
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,3,0.039182400703430174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,7,0.045368000864982605
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,7,0.041038399934768675
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,15,0.045281600952148435
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,15,0.041140800714492796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,31,0.045326399803161624
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,31,0.039699199795722964
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,63,0.04744159877300262
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,63,0.03916000127792359
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,127,0.04990400075912475
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,127,0.04323999881744385
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,255,0.08034719824790955
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,255,0.050292801856994626
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,511,0.1497007966041565
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,511,0.08398399949073791
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,1,0.0104592002928257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,float16,1023,0.2855423927307129
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,1,0.008441600203514098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,4,2,128,1,float16,fp8,1023,0.15383360385894776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,3,0.00835840031504631
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,3,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,7,0.008395200222730636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,7,0.010414399951696397
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,15,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,15,0.01037440001964569
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,31,0.010284800082445145
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,31,0.00952960029244423
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,63,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,63,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,127,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,127,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,255,0.010462400317192078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,255,0.010372799634933472
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,511,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,511,0.010387200117111205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,1023,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,1023,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,2047,0.011579199880361556
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,2047,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,4095,0.012428800016641617
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,4095,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,8191,0.014585599303245544
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,8191,0.012462399899959564
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,16383,0.015387199819087982
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,16383,0.030537599325180055
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,32767,0.04729920029640198
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,32767,0.030555200576782227
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,1,0.0083856001496315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,fp8,65535,0.04763999879360199
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,1,128,1,float16,float16,65535,0.08203200101852418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,1,0.008422400057315826
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,3,0.008463999629020691
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,3,0.008451200276613235
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,7,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,7,0.008425600081682205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,15,0.00926240012049675
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,15,0.008425600081682205
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,31,0.008379200100898742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,31,0.008500800281763077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,63,0.008455999940633774
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,127,0.010353600233793258
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,63,0.008459199965000153
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,127,0.008491200208663941
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,255,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,255,0.008403199911117553
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,511,0.00841279998421669
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,511,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,1023,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,1023,0.009110400080680847
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,2047,0.010467199981212616
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,2047,0.01035040020942688
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,4095,0.012531200051307678
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,4095,0.012379200011491776
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,8191,0.014505599439144135
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,8191,0.027187201380729675
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,16383,0.04519200026988983
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,16383,0.0277072012424469
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,32767,0.07841280102729797
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,32767,0.04549919962882996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,1,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,fp8,65535,0.08116000294685363
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,4,2,128,1,float16,float16,65535,0.14617760181427003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,1,0.008537600189447403
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,7,0.010204800218343735
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,3,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,3,0.009646400064229965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,7,0.00843999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,15,0.009835200011730194
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,15,0.010539200156927109
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,31,0.010332799702882766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,31,0.009415999799966813
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,63,0.008505599945783615
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,63,0.010524799674749374
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,127,0.010331200063228607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,127,0.00934239998459816
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,255,0.010345599800348281
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,255,0.009600000083446502
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,511,0.010406400263309478
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,1023,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,511,0.008873599767684936
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,1023,0.01037919968366623
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,2047,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,2047,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,4095,0.028046399354934692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,4095,0.014539200067520141
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,8191,0.028865599632263185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,8191,0.04553920030593872
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,16383,0.07964479923248291
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,16383,0.045495998859405515
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,32767,0.08016160130500793
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,32767,0.14810880422592163
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,65535,0.2841072082519531
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,65535,0.14832639694213867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,1,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,1,0.008379200100898742
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,3,0.00846719965338707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,3,0.008454400300979614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,float16,131071,0.5562863826751709
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,7,0.00841120034456253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,64,1,2,1,128,1,float16,fp8,131071,0.2854144096374512
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,15,0.008814399689435959
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,31,0.009414400160312652
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,15,0.008479999750852585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,7,0.010355199873447418
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,31,0.008441600203514098
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,63,0.008795200288295746
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,63,0.010470400005578995
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,127,0.00965920016169548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,127,0.008452799916267396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,255,0.010555200278759003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,255,0.00843999981880188
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,511,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,1023,0.00968799963593483
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,511,0.010440000146627427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,1023,0.008484800159931184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,2047,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,2047,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,4095,0.010500799864530563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,4095,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,8191,0.012670400738716125
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,8191,0.012572799623012543
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,16383,0.014545600116252898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,16383,0.016763199865818024
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,32767,0.01852640062570572
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,32767,0.016569599509239197
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,65535,0.020571200549602507
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,65535,0.018628799915313722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,float16,131071,0.02096640020608902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1,1,2,1,128,1,float16,fp8,131071,0.020734399557113647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,1,0.008454400300979614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,1,0.008396799862384795
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,3,0.010363200306892395
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,3,0.00851999968290329
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,7,0.010395199805498124
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,7,0.008814399689435959
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,15,0.010315199941396713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,15,0.008399999886751174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,31,0.010390400141477584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,31,0.008454400300979614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,63,0.009895999729633332
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,63,0.010369600355625152
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,127,0.010334400087594986
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,127,0.008392000198364257
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,255,0.010332799702882766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,255,0.008551999926567078
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,511,0.008416000008583068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,511,0.010476800054311753
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,1023,0.008433599770069123
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,1023,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,2047,0.01135680004954338
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,4095,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,2047,0.010454399883747101
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,8191,0.012646399438381195
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,4095,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,8191,0.0126351997256279
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,16383,0.018595199286937713
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,32767,0.018011200428009033
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,16383,0.016651199758052827
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,32767,0.018512000143527985
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,65535,0.02075680047273636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,65535,0.018855999410152435
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,float16,131071,0.03808639943599701
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,1,0.00838399976491928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,1,0.008564800024032593
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2,1,2,1,128,1,float16,fp8,131071,0.022920000553131103
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,3,0.008374399691820144
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,3,0.008507200330495835
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,7,0.0083856001496315
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,15,0.010351999849081039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,7,0.008656000345945358
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,15,0.008595199882984161
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,31,0.010331200063228607
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,63,0.008476799726486206
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,31,0.008703999966382981
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,63,0.00841120034456253
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,127,0.008393599838018417
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,255,0.009505599737167358
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,127,0.008544000238180161
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,511,0.008606400340795517
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,255,0.008475200086832047
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,511,0.008553600311279297
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,1023,0.010318399965763092
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,1023,0.008419200032949447
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,2047,0.010358399897813796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,2047,0.010755199939012527
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,4095,0.01252480000257492
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,4095,0.01058719977736473
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,8191,0.014556799829006196
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,16383,0.045278400182724
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,8191,0.0272816002368927
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,16383,0.0281360000371933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,32767,0.07972000241279602
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,32767,0.04521760046482086
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,65535,0.14701440334320068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,65535,0.08068959712982178
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,1,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,float16,131071,0.2844223976135254
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,1,0.008550400286912918
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,32,1,2,1,128,1,float16,fp8,131071,0.14800959825515747
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,3,0.00865280032157898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,3,0.009424000233411788
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,7,0.008452799916267396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,7,0.008416000008583068
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,15,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,31,0.008432000130414962
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,15,0.008470399677753449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,63,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,63,0.008638399839401244
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,31,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,127,0.01040000021457672
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,255,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,127,0.008737599849700928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,255,0.008444800227880477
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,511,0.010419200360774993
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,1023,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,511,0.009921599924564362
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,1023,0.010390400141477584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,2047,0.010409600287675857
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,2047,0.011079999804496764
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,4095,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,4095,0.010555200278759003
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,8191,0.014575999975204468
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,8191,0.014550399780273438
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,16383,0.016577599942684172
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,16383,0.015772800147533416
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,32767,0.01865759938955307
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,32767,0.0165583997964859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,65535,0.035020801424980166
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,65535,0.018603199720382692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,float16,131071,0.052908802032470705
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,1,0.008423999696969987
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,1,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,4,1,2,1,128,1,float16,fp8,131071,0.03505280017852783
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,3,0.008369600027799606
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,3,0.008353599905967712
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,7,0.008513599634170532
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,15,0.00846719965338707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,7,0.008452799916267396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,15,0.008500800281763077
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,31,0.008419200032949447
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,31,0.008390399813652038
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,63,0.008504000306129456
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,63,0.008436799794435502
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,127,0.008724799752235413
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,127,0.008489599823951722
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,255,0.008534400165081025
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,255,0.008380799740552902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,511,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,1023,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,511,0.010265599936246872
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,1023,0.01051200032234192
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,2047,0.026855999231338502
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,2047,0.012564800679683685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,4095,0.026910400390625
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,4095,0.04424799978733063
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,8191,0.07857440114021301
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,8191,0.04496319890022278
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,16383,0.07910400032997131
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,16383,0.14707520008087158
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,32767,0.14810080528259278
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,32767,0.2830159902572632
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,1,0.010532800108194351
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,1,0.010644800215959548
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,3,0.010520000010728836
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,fp8,65535,0.2864255905151367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,3,0.010596799850463866
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,128,1,2,1,128,1,float16,float16,65535,0.5559631824493408
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,7,0.01048479974269867
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,7,0.010463999956846238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,15,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,31,0.010342399775981902
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,31,0.010692799836397171
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,15,0.010491199791431427
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,63,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,127,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,63,0.010444799810647965
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,127,0.010360000282526016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,255,0.010436800122261048
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,511,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,511,0.011785600334405899
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,255,0.010660800337791442
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,1023,0.027294400334358215
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,1023,0.014569599926471711
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,2047,0.04543839991092682
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,2047,0.028193598985671996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,4095,0.07997599840164185
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,4095,0.04569759964942932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,8191,0.14860320091247559
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,8191,0.08151839971542359
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,1,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,1,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,float16,16383,0.28649120330810546
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,3,0.01247519999742508
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,256,1,2,1,128,1,float16,fp8,16383,0.15066560506820678
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,7,0.01252799928188324
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,7,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,15,0.012491200119256973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,3,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,15,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,31,0.01250080019235611
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,63,0.012534399330615998
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,31,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,63,0.01249760016798973
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,127,0.012449599802494049
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,127,0.012489599734544754
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,255,0.012601600587368011
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,255,0.01242239996790886
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,511,0.02799839973449707
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,511,0.014545600116252898
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,1023,0.04536159932613373
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,1023,0.029129600524902342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,2047,0.046804800629615784
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,2047,0.08100320100784301
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,4095,0.15175199508666992
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,4095,0.08183040022850037
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,1,0.010486400127410889
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,1,0.010420800000429154
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,float16,8191,0.29342079162597656
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,3,0.010447999835014344
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,512,1,2,1,128,1,float16,fp8,8191,0.15175679922103882
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,7,0.010329599678516387
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,15,0.010375999659299851
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,3,0.008406399935483932
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,7,0.009988799691200256
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,31,0.010396800190210342
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,15,0.008446399867534638
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,31,0.0104032002389431
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,63,0.01040479987859726
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,63,0.008459199965000153
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,127,0.010380800068378448
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,127,0.008796799927949905
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,255,0.010424000024795533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,255,0.01037440001964569
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,511,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,511,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,1023,0.010441599786281586
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,2047,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,2047,0.010435199737548828
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,4095,0.012564800679683685
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,1023,0.010500799864530563
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,4095,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,8191,0.014571200311183929
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,8191,0.01300320029258728
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,16383,0.014535999298095703
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,32767,0.031518399715423584
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,16383,0.016731199622154237
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,32767,0.01658560037612915
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,65535,0.049377599358558656
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,float16,131071,0.08416960239410401
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,65535,0.03191519975662231
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,8,1,2,1,128,1,float16,fp8,131071,0.04943360090255737
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,3,0.0165583997964859
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,1,0.016571199893951415
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,1,0.017342400550842286
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,3,0.016603200137615202
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,7,0.01828960031270981
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,7,0.016465599834918975
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,15,0.016675199568271636
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,31,0.016763199865818024
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,15,0.016568000614643096
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,31,0.016574400663375854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,63,0.01687999963760376
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,127,0.017071999609470367
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,63,0.016840000450611115
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,127,0.016540800034999848
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,255,0.02744159996509552
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,511,0.04543040096759796
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,255,0.016604800522327424
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,511,0.028905600309371948
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,1023,0.08013439774513245
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,1023,0.04697760045528412
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,2047,0.14806400537490844
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,2047,0.08209760189056396
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,1,0.02693600058555603
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,float16,4095,0.28460800647735596
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,1,0.02491839975118637
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,1024,1,2,1,128,1,float16,fp8,4095,0.15118240118026732
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,3,0.024774399399757386
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,3,0.026745599508285523
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,7,0.02686559855937958
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,7,0.024823999404907225
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,15,0.0247856006026268
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,31,0.026881599426269533
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,15,0.0268640011548996
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,31,0.02497600018978119
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,63,0.026910400390625
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,63,0.024806399643421174
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,127,0.03054080009460449
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,127,0.024977600574493407
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,255,0.030926400423049928
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,255,0.045484799146652224
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,511,0.08022239804267883
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,511,0.047409600019454955
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,1023,0.14844160079956054
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,1023,0.08302559852600097
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,1,0.010417599976062775
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,1,0.010398399829864503
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,float16,2047,0.28507840633392334
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,2048,1,2,1,128,1,float16,fp8,2047,0.15120960474014283
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,3,0.010428799688816071
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,3,0.010412800312042236
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,7,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,7,0.010385599732398988
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,15,0.010377600044012069
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,15,0.010446400195360184
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,31,0.01048159971833229
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,31,0.010360000282526016
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,63,0.010401599854230881
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,63,0.010393600165843963
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,127,0.010438399761915207
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,127,0.010416000336408614
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,255,0.010478399693965912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,255,0.009185600280761718
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,511,0.01043040007352829
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,511,0.010332799702882766
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,1023,0.010407999902963639
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,1023,0.010367999970912933
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,2047,0.010807999968528747
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,4095,0.012620800733566284
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,2047,0.010427200049161912
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,4095,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,8191,0.014697599411010741
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,8191,0.012495999783277511
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,16383,0.030220800638198854
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,16383,0.016487999260425566
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,32767,0.047279998660087585
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,32767,0.030803200602531434
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,65535,0.08182880282402039
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,65535,0.04765760004520416
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,float16,131071,0.1500383973121643
SGLang,0.5.8.post1,NVIDIA GB200,generation_attention,trtllm_mha,16,1,2,1,128,1,float16,fp8,131071,0.08281440138816834
