framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,window_size,beam_width,attn_dtype,kv_cache_dtype,step,latency
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,1,0.023919999599456787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,3,0.023823998868465424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,1,0.02367466688156128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,7,0.0236160010099411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,3,0.023845332364241283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,7,0.023386667172114056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,15,0.025226667523384094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,15,0.024186665813128155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,31,0.025285333395004272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,31,0.025045332809289295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,63,0.025450666745503742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,63,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,127,0.025237334271272022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,127,0.02510400116443634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,255,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,255,0.025061334172884624
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,511,0.03764266769091288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,511,0.03362133353948593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,1023,0.05580799778302511
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,1023,0.04991999765237173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,2047,0.08447466293970744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,2047,0.09628267089525859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,fp8,4095,0.14434666434923807
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,64,0,1,float16,float16,4095,0.16936532656351724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,1,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,1,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,3,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,15,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,31,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,31,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,63,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,63,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,127,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,255,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,511,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,511,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,1023,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,1023,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,2047,0.014389333625634512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,2047,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,4095,0.01682666689157486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,4095,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,8191,0.02143999934196472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,8191,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,16383,0.03525333354870478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,fp8,16383,0.029370665550231934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,1,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,3,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,3,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,7,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,7,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,15,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,31,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,64,0,1,float16,float16,7,0.01071999967098236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,63,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,63,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,127,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,127,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,255,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,511,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,511,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,1023,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,1023,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,2047,0.017642666896184284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,2047,0.016986666868130367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,4095,0.022991999983787537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,4095,0.019189332922299702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,8191,0.03583466758330663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,8191,0.029301332930723827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,16383,0.04836266736189524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,float16,16383,0.0561653325955073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,64,0,1,float16,fp8,255,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,1,0.037434667348861694
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,1,0.03742400060097376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,3,0.0377866675456365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,3,0.03770133356253306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,7,0.038933334251244865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,7,0.03977066775163015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,31,0.03966933240493139
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,15,0.03770133356253306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,15,0.03751999884843826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,31,0.03793066740036011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,63,0.037685332198937736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,63,0.03745066622893015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,127,0.03801066676775614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,255,0.039936001102129616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,127,0.037978666524092354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,255,0.03734400123357773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,511,0.060565332571665444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,511,0.05640000104904175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,1023,0.09604799747467041
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,1023,0.08528000116348267
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,float16,2047,0.1700106660525004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,1,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,1,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,3,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,3,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,7,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,7,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,15,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,15,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,31,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,31,0.012768000364303589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,63,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,63,0.012650666137536367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,127,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,255,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,511,0.014074667046467463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,511,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,1023,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,1023,0.016773333152135212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,2047,0.023957334458827972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,2047,0.023530667026837666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,4095,0.03899200012286504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,4095,0.0315733328461647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,8191,0.0602400004863739
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,8191,0.05207466582457224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,float16,16383,0.10170666376749675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,64,0,1,float16,fp8,16383,0.08715200424194336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,1,0.06726400057474773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,1,0.0641546646753947
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,3,0.0668213317791621
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,3,0.06437866886456807
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,7,0.06692799925804138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,7,0.0644053320089976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,64,0,1,float16,fp8,2047,0.14661866426467896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,15,0.06679466863473256
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,15,0.06433066725730896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,31,0.06459199885527293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,63,0.06628266473611195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,63,0.06489600241184235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,127,0.06849599877993266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,127,0.06437333424886067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,255,0.06849599877993266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,31,0.06663999954859416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,255,0.06637333333492279
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,511,0.107013334830602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,511,0.09731733798980713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,float16,1023,0.17717333634694418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,64,0,1,float16,fp8,1023,0.15653333067893982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,1,0.12598933776219687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,1,0.11995733777681987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,3,0.1197119951248169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,3,0.12573867042859396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,7,0.11997866630554199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,7,0.12693867087364197
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,15,0.11973333358764648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,15,0.12599999705950418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,31,0.12622400124867758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,63,0.12568533420562744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,63,0.11965333422025044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,31,0.12052800258000691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,127,0.12409067153930664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,127,0.12184000015258789
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,float16,255,0.12480533123016357
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,64,0,1,float16,fp8,255,0.11979200442632039
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,1,0.23923200368881226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,1,0.23059733708699545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,3,0.2395520011583964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,3,0.23083200057347616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,7,0.23009600241978964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,7,0.23857067028681436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,15,0.2395520011583964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,15,0.2306506633758545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,31,0.23883734146753946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,31,0.23036267360051474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,63,0.24034667015075684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,1,0.01481066644191742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,63,0.23067200183868408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,1,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,3,0.014618666221698126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,3,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,float16,127,0.23502933979034424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,64,0,1,float16,fp8,127,0.2323519984881083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,7,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,7,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,15,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,15,0.014581333845853806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,31,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,63,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,63,0.014602666099866232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,31,0.015450666348139444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,127,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,127,0.013994666437307993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,255,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,255,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,511,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,511,0.016965333372354507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,1023,0.02146133283774058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,1023,0.021274665991465252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,2047,0.039690665900707245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,2047,0.031925333042939506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,4095,0.060533334811528526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,4095,0.05374933282534281
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,8191,0.10366933544476827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,8191,0.08727467060089111
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,float16,16383,0.18598399559656778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,64,0,1,float16,fp8,16383,0.15523200233777365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,1,0.46613868077596027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,1,0.449285348256429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,3,0.46593066056569415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,3,0.44966399669647217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,7,0.4655253489812215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,7,0.44891734917958576
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,15,0.4663360118865967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,15,0.4489813248316447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,31,0.4659786621729533
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,31,0.44938135147094727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,fp8,63,0.44946134090423584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,64,0,1,float16,float16,63,0.46587200959523517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,1,0.919269323348999
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,1,0.8841973145802816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,3,0.9189600149790446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,3,0.8861333529154459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,7,0.9192266464233398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,7,0.8844426472981771
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,15,0.9199360211690267
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,1,0.017157333592573803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,1,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,3,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,15,0.8857866923014323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,3,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,7,0.017680000513792038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,7,0.017466666797796886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,15,0.017530667285124462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,15,0.017344000438849132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,31,0.01752000053723653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,31,0.01746133342385292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,63,0.017263999829689663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,63,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,127,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,127,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,float16,31,0.9198506673177084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,255,0.017466666797796886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,255,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,64,0,1,float16,fp8,31,0.8850239912668864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,511,0.02162666618824005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,511,0.021530665457248688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,1023,0.03580799947182337
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,1023,0.029317334294319153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,2047,0.056032001972198486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,2047,0.0498933345079422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,4095,0.09379733602205913
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,4095,0.0810346653064092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,fp8,8191,0.1406826674938202
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,64,0,1,float16,float16,8191,0.16715733210245767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,1,0.025263999899228413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,1,0.0276053324341774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,3,0.025605333348115284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,3,0.027306665976842243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,7,0.027424000203609467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,7,0.025573333104451496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,15,0.027471999327341717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,15,0.025546667476495106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,31,0.02731200059254964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,31,0.025557334224383037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,63,0.02755733331044515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,63,0.025301332275072735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,127,0.027424000203609467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,127,0.025701334079106648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,255,0.03242666771014532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,255,0.02532266577084859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,511,0.05221333106358846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,511,0.03770666569471359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,1023,0.09018666545550029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,1023,0.054048001766204834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,2047,0.16910399993260702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,2047,0.09449600179990132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,1,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,fp8,4095,0.166810671488444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,3,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,64,128,0,1,float16,float16,4095,0.3206026752789815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,3,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,7,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,7,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,15,0.010677333921194077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,31,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,63,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,127,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,255,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,511,0.01179733375708262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,511,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,1023,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,1023,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,2047,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,2047,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,4095,0.019109333554903667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,4095,0.01704000060757001
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,8191,0.035749333600203194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,8191,0.021157334248224895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,1,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,float16,16383,0.05622399846712748
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,64,128,0,1,float16,fp8,16383,0.033546666304270424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,3,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,7,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,7,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,15,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,31,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,63,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,63,0.012666666259368261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,127,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,255,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,255,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,511,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,511,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,1023,0.014671999961137772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,1023,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,2047,0.019509332875410717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,2047,0.016986666868130367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,4095,0.0354666660229365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,4095,0.021551998953024547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,8191,0.055530667304992676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,8191,0.035418666899204254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,1,0.0458133320013682
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,fp8,16383,0.05435733497142792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,64,128,0,1,float16,float16,16383,0.09776000181833903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,1,0.039503999054431915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,3,0.04553066690762838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,3,0.03986666599909464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,7,0.04552533229192098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,7,0.03973866750796636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,15,0.04590400060017904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,15,0.03959999978542328
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,31,0.04598933458328247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,63,0.04562133550643921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,31,0.039781334499518074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,63,0.039450667798519135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,127,0.04784533381462097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,127,0.03982399900754293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,255,0.05236266553401947
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,255,0.04156800111134847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,511,0.09045333663622539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,511,0.059994667768478394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,1023,0.16472533345222473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,1023,0.09288533528645833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,1,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,float16,2047,0.31762667496999103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,1,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,64,128,0,1,float16,fp8,2047,0.1675893266995748
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,3,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,3,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,7,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,7,0.012698666503032049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,15,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,15,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,31,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,31,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,63,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,63,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,127,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,127,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,255,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,255,0.012725333372751871
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,511,0.014922666052977243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,511,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,1023,0.01728533332546552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,1023,0.01695466662446658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,2047,0.037920000652472176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,2047,0.02325333406527837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,4095,0.05857066810131073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,4095,0.03761066744724909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,8191,0.05966933568318685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,8191,0.10089600086212158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,float16,16383,0.1821440060933431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,64,128,0,1,float16,fp8,16383,0.1000373363494873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,1,0.08245866497357686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,3,0.08099733293056488
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,1,0.07028799752394359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,3,0.0687360018491745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,7,0.08241066833337148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,7,0.06849066913127899
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,15,0.07050666709740956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,31,0.08125866452852885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,15,0.08348799745241801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,31,0.06835199892520905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,63,0.0825973351796468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,63,0.06838933130105336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,127,0.0825493335723877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,127,0.07070399820804596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,255,0.07019733389218648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,255,0.09116799632708232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,511,0.16588800152142844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,511,0.10648000240325928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,float16,1023,0.31254400809605914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,64,128,0,1,float16,fp8,1023,0.17160000403722128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,1,0.15160000324249268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,1,0.13088533282279968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,3,0.15130133430163065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,3,0.1311199963092804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,7,0.1509866714477539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,7,0.12997866670290628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,15,0.1519306699434916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,15,0.13034133116404215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,31,0.15179733435312906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,31,0.13052266836166382
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,63,0.15092800060908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,63,0.13036266962687174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,127,0.15130133430163065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,float16,255,0.1690666675567627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,127,0.12903466820716858
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,64,128,0,1,float16,fp8,255,0.12894399960835776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,1,0.2914453347524007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,1,0.24739199876785278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,3,0.292192002137502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,3,0.2480213244756063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,7,0.29053332408269245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,7,0.24852800369262695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,15,0.29183467229207355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,15,0.24701333045959473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,31,0.29125332832336426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,31,0.24827200174331665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,63,0.29202133417129517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,63,0.24823466936747232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,1,0.016154666741689045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,float16,127,0.28826133410135907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,1,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,64,128,0,1,float16,fp8,127,0.24683199326197305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,3,0.015647999942302704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,3,0.015114666273196539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,7,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,15,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,7,0.014720000326633453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,15,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,31,0.014896000425020853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,31,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,63,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,63,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,127,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,127,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,255,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,255,0.014720000326633453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,511,0.018719999740521114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,511,0.017349333812793095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,1023,0.034341332813103996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,1023,0.023237332701683044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,2047,0.05889600018660227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,2047,0.03991466760635376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,4095,0.0602453351020813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,4095,0.10058133800824483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,8191,0.18342934052149454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,8191,0.10178666313489278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,fp8,16383,0.1831093430519104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,64,128,0,1,float16,float16,16383,0.34990934530893963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,1,0.48184001445770264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,1,0.5683253208796183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,3,0.5683626731236776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,3,0.48163731892903644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,7,0.4814079999923706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,7,0.5683573484420776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,15,0.4815680185953776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,15,0.5683679978052775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,31,0.48127468427022296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,31,0.5685706535975138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,fp8,63,0.4819093147913615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,64,128,0,1,float16,float16,63,0.5683093468348185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,1,0.9494986534118652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,1,1.1222879886627197
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,3,0.9481546878814697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,3,1.1221760114034016
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,7,0.9487786293029785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,7,1.1239039897918701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,15,0.9487040042877197
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,15,1.1220906575520833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,1,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,1,0.017792000124851864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,3,0.01923199991385142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,3,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,7,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,7,0.01758933315674464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,15,0.019226666539907455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,15,0.01886933296918869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,31,0.01912533367673556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,31,0.018911999960740406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,63,0.01913600042462349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,63,0.0189280000825723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,127,0.019167999426523846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,127,0.019088000059127808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,255,0.019253333409627277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,255,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,511,0.032373333970705666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,511,0.023002666731675465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,fp8,31,0.9495093027750651
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,64,128,0,1,float16,float16,31,1.122704029083252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,1023,0.052789335449536644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,1023,0.03444266567627589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,2047,0.05597866574923197
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,2047,0.09339732925097148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,4095,0.17059199015299478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,4095,0.0934933324654897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,1,0.021327999730904896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,float16,8191,0.3210879961649577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,64,128,0,1,float16,fp8,8191,0.1662453313668569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,1,0.021536000072956085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,3,0.021418665846188862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,3,0.021525333325068157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,7,0.02143999934196472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,7,0.021189334491888683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,15,0.02179199953873952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,15,0.02160000056028366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,31,0.021226666867733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,31,0.021488000949223835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,63,0.021173333128293354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,63,0.021189334491888683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,127,0.021205333371957142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,127,0.021130666136741638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,255,0.021221332252025604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,255,0.021712000171343487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,511,0.031040000418821972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,511,0.02815466622511546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,1023,0.04155733436346054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,1023,0.04651733239491781
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,2047,0.07986666758855183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,2047,0.06915733218193054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,1,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,float16,4095,0.13819199800491333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,64,0,1,float16,fp8,4095,0.11753599842389424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,3,0.011829332758982977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,7,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,7,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,15,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,15,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,31,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,31,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,63,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,63,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,127,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,255,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,255,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,511,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,511,0.013514666507641474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,1023,0.013482666263977686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,1023,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,2047,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,2047,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,4095,0.01676799977819125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,4095,0.01807466646035512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,8191,0.01988799994190534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,8191,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,float16,16383,0.029440000653266907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,1,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,64,0,1,float16,fp8,16383,0.025546667476495106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,7,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,15,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,31,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,15,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,127,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,127,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,255,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,255,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,511,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,511,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,1023,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,1023,0.01339200014869372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,2047,0.01682666689157486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,2047,0.015530666957298914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,4095,0.021354667842388153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,4095,0.021194666624069214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,8191,0.033520000676314034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,8191,0.029274667302767437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,1,0.03186666717131933
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,float16,16383,0.055914665261904396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,64,0,1,float16,fp8,16383,0.04806933303674062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,1,0.031301334500312805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,3,0.03148266673088074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,3,0.03149333347876867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,7,0.0315786674618721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,7,0.03154666721820831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,15,0.031514666974544525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,15,0.03123733401298523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,31,0.03151999910672506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,31,0.03179199993610382
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,63,0.03136533250411352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,63,0.03126399964094162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,127,0.03150933235883713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,127,0.0314026673634847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,255,0.03363200028737386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,255,0.031914666295051575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,511,0.04987200101216634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,511,0.04548266530036926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,1023,0.06850133339564006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,1023,0.07760000228881836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,float16,2047,0.1349066694577535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,1,0.01190399999419848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,64,0,1,float16,fp8,2047,0.1179200013478597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,1,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,3,0.011594666788975397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,3,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,7,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,7,0.013637332866589228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,15,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,15,0.012261333564917246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,31,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,31,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,63,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,63,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,127,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,127,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,255,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,511,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,255,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,511,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,1023,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,1023,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,2047,0.023247999449570973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,2047,0.021269333859284718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,4095,0.03590933233499527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,4095,0.029391999046007793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,8191,0.058778668443361916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,8191,0.05017066498597463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,float16,16383,0.1015786627928416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,1,0.05228800078233083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,64,0,1,float16,fp8,16383,0.08630399902661641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,1,0.052111998200416565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,3,0.05190399785836538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,3,0.05227733155091604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,7,0.05250666538874308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,7,0.05182399849096934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,15,0.05212800204753876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,15,0.05195199946562449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,31,0.05309866865475973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,31,0.051776001850763954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,63,0.05388266841570536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,63,0.05193600058555603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,127,0.05418133238951365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,127,0.052149335543314614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,255,0.05444266895453135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,255,0.05248000224431356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,511,0.08480532964070638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,511,0.07670400043328603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,float16,1023,0.13827199737230936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,1,0.0969493289788564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,64,0,1,float16,fp8,1023,0.12176000078519185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,1,0.09277333815892537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,3,0.09107733766237895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,7,0.09719467163085938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,3,0.09826667110125224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,7,0.0928053359190623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,15,0.09716266393661499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,15,0.09131733576456706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,31,0.09797333677609761
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,31,0.09289600451787312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,63,0.09100799759229024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,63,0.09709866841634114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,127,0.09468799829483032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,127,0.09725333253542583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,fp8,255,0.09301867087682088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,64,0,1,float16,float16,255,0.09707199533780415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,1,0.18336532513300577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,3,0.17719467480977377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,3,0.18291733662287393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,1,0.17702933152516684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,7,0.1832480033238729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,7,0.17706133921941122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,15,0.18314667542775473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,15,0.17734400431315103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,31,0.18308266003926596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,31,0.17685333887736002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,63,0.1835199991861979
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,1,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,63,0.17732266585032144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,1,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,3,0.013525333255529404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,3,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,7,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,fp8,127,0.17781333128611246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,64,0,1,float16,float16,127,0.1823413372039795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,7,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,15,0.013381333400805792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,15,0.013503999759753546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,31,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,31,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,63,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,63,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,127,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,127,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,255,0.013530666629473368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,255,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,511,0.015306666493415833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,511,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,1023,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,1023,0.019365333020687103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,2047,0.03196800003449122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,2047,0.027429332335789997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,4095,0.04293333490689596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,4095,0.05105599761009216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,8191,0.07051200171311696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,8191,0.08178133269151051
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,float16,16383,0.14434666434923807
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,64,0,1,float16,fp8,16383,0.12199466427167256
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,1,0.35594133536020917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,3,0.35497601826985675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,1,0.34160534540812176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,3,0.3415466547012329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,7,0.3549493153889974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,7,0.3415893316268921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,15,0.35440532366434735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,15,0.34115199247996014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,31,0.35576534271240234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,float16,63,0.354912002881368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,31,0.3415093421936035
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,64,0,1,float16,fp8,63,0.3415786822636922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,1,0.6963573296864828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,1,0.6669279734293619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,3,0.6677760283152262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,3,0.6966506640116373
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,7,0.695904016494751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,7,0.667744000752767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,15,0.6954399744669596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,1,0.017077332983414333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,1,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,15,0.6682613690694174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,3,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,3,0.01749333366751671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,7,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,7,0.015578666081031164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,15,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,float16,31,0.6963733037312826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,15,0.01543466622630755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,31,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,31,0.01570133368174235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,63,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,64,0,1,float16,fp8,31,0.6670933564503988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,63,0.017050666113694508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,127,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,127,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,255,0.01704000060757001
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,255,0.01540800059835116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,511,0.021295999487241108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,511,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,1023,0.029743999242782593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,1023,0.025477332373460133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,2047,0.050426666935284935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,2047,0.044213334719340004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,4095,0.08260799944400787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,4095,0.07123733560244243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,1,0.02313599983851115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,1,0.02293866624434789
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,float16,8191,0.14589866995811462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,64,0,1,float16,fp8,8191,0.12201066811879475
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,3,0.0233599990606308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,3,0.02223466585079829
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,7,0.02346666653951009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,7,0.021375998854637146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,15,0.02309866746266683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,15,0.02292266736427943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,31,0.023152001202106476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,31,0.022778667509555817
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,63,0.023205332458019257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,63,0.022810667753219604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,127,0.023333333432674408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,127,0.021589333812395733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,255,0.025055999557177227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,255,0.02126399924357732
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,511,0.04317333300908407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,511,0.03148266673088074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,1023,0.04570133487383524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,1023,0.07256533205509186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,2047,0.07784000039100647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,2047,0.1338879962762197
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,1,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,fp8,4095,0.13436800241470337
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,1,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,48,48,128,0,1,float16,float16,4095,0.2512800097465515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,3,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,7,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,7,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,15,0.012042666474978128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,15,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,31,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,63,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,63,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,127,0.010677333921194077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,127,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,255,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,511,0.012256000190973282
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,511,0.012693333129088083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,1023,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,1023,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,2047,0.017194667210181553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,2047,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,4095,0.018794666975736618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,4095,0.01699200024207433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,8191,0.025616000096003216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,8191,0.020986666282018025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,1,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,float16,16383,0.045040001471837364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,1,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,48,48,128,0,1,float16,fp8,16383,0.0295413335164388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,3,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,7,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,7,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,15,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,15,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,31,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,63,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,31,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,63,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,127,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,127,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,255,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,511,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,511,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,1023,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,1023,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,2047,0.01800000046690305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,2047,0.017194667210181553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,4095,0.029616000751654308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,4095,0.0215786670645078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,8191,0.0525546669960022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,8191,0.03335466732581457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,float16,16383,0.08943466345469157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,48,48,128,0,1,float16,fp8,16383,0.05465066432952881
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,1,0.03594133257865906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,3,0.03576533248027166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,1,0.033733333150545754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,3,0.03356799980004629
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,7,0.03573866685231527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,7,0.03381866713364919
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,15,0.03818133225043615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,15,0.03332799921433131
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,31,0.037621334195137024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,63,0.03568533311287562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,31,0.03329066683848699
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,63,0.03359466542800268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,127,0.037861332297325134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,127,0.03366400053103765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,255,0.043791999419530235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,255,0.033786666889985405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,511,0.07216533521811168
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,1023,0.126991997162501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,511,0.05128000179926554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,1023,0.07498666644096375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,1,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,fp8,2047,0.1322933336098989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,48,48,128,0,1,float16,float16,2047,0.24262932936350504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,3,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,1,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,3,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,7,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,7,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,15,0.013797332843144735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,31,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,31,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,63,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,63,0.013445333888133367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,127,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,127,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,255,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,511,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,255,0.012752000242471695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,511,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,1023,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,1023,0.017077332983414333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,2047,0.02329600105683009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,2047,0.03075733284155528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,4095,0.0563679983218511
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,4095,0.03647999962170919
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,8191,0.05799466868241628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,8191,0.09489066402117412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,1,0.06411733229955037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,1,0.05413866539796194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,float16,16383,0.1728853384653727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,48,48,128,0,1,float16,fp8,16383,0.0997759997844696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,3,0.054287999868392944
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,3,0.06413333117961884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,7,0.0642133355140686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,7,0.05397866666316986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,15,0.06413333117961884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,15,0.05442666510740916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,31,0.06452266871929169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,31,0.05389333268006643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,63,0.06442133088906606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,63,0.05400000015894572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,127,0.06404800216356914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,127,0.0562666654586792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,255,0.056176001826922096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,255,0.0724373310804367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,511,0.12797333796819052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,511,0.08284799754619598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,float16,1023,0.23755733172098795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,48,48,128,0,1,float16,fp8,1023,0.1332480013370514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,1,0.11756799618403117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,1,0.10107733805974324
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,3,0.11741866668065389
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,3,0.10098133484522502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,7,0.11744532982508342
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,7,0.10106666882832845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,15,0.1179200013478597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,15,0.10116266210873921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,31,0.11760000387827556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,31,0.10143466790517171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,63,0.11782399813334148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,63,0.09976533055305481
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,127,0.11716266473134358
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,127,0.1013706624507904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,float16,255,0.13065066933631897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,48,48,128,0,1,float16,fp8,255,0.10220266381899516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,1,0.22237332661946616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,1,0.18993600209554037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,3,0.2204213341077169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,3,0.18988800048828125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,7,0.22198933362960815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,7,0.1894879937171936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,15,0.22222399711608887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,15,0.19062399864196777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,31,0.2205066680908203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,31,0.19016534090042114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,63,0.22218134005864462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,63,0.18941867351531982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,1,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,1,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,3,0.01479999969402949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,fp8,127,0.18950400749842325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,3,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,7,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,48,48,128,0,1,float16,float16,127,0.2206666668256124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,15,0.014592000593741735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,7,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,15,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,31,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,63,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,63,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,31,0.013967999567588171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,127,0.014186666657527288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,127,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,255,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,255,0.013621332744757334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,511,0.016976000120242436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,511,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,1023,0.023925334215164185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,1023,0.01958400011062622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,2047,0.048485333720842995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,2047,0.031530665854612984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,4095,0.07871466875076294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,4095,0.04939733445644379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,8191,0.07932800054550171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,8191,0.1416000028451284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,float16,16383,0.26614399751027423
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,48,48,128,0,1,float16,fp8,16383,0.1420799990495046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,1,0.3646719853083293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,1,0.4285866816838582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,3,0.42898134390513104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,3,0.36557332674662274
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,7,0.3651839892069499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,7,0.4278133312861125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,15,0.42935999234517414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,15,0.36577598253885907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,31,0.36473600069681805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,31,0.42884798844655353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,fp8,63,0.3657493193944295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,48,48,128,0,1,float16,float16,63,0.42923200130462646
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,1,0.7169600327809652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,1,0.8442986806233724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,3,0.7196640173594157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,3,0.8451039791107178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,7,0.7165813446044922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,7,0.8460266590118408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,15,0.7198560237884521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,15,0.8455039660135905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,1,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,1,0.017008000363906223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,3,0.01757866640885671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,3,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,7,0.017263999829689663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,7,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,15,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,15,0.01687466725707054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,31,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,fp8,31,0.7170080343882242
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,48,48,128,0,1,float16,float16,31,0.8449386755625407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,31,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,63,0.016901332885026932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,63,0.017717332889636356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,127,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,127,0.01720000058412552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,255,0.017781333376963932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,255,0.016143999993801117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,511,0.024901332954565685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,511,0.02143466720978419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,1023,0.045647998650868736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,1023,0.029509333272775013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,2047,0.08035733302434285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,2047,0.05004799862702688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,4095,0.14285866419474283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,4095,0.08165866633256276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,fp8,8191,0.14269866545995077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,48,48,128,0,1,float16,float16,8191,0.26985599597295123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,1,0.01907733331123988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,1,0.019066666563351948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,3,0.019365333020687103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,3,0.019445333629846573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,7,0.019354666272799175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,15,0.0191040001809597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,7,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,15,0.01932799940307935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,31,0.019365333020687103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,31,0.01899733394384384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,63,0.019002666076024372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,127,0.01933866615096728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,63,0.019493332753578823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,127,0.01950399950146675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,255,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,255,0.019280000279347103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,511,0.025653332471847534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,511,0.024832000335057575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,1023,0.040522667268911995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,1023,0.034245334565639496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,2047,0.0670773337284724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,2047,0.060592000683148704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,float16,4095,0.11575999855995178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,1,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,1,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,64,0,1,float16,fp8,4095,0.09825600186983745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,3,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,7,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,31,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,31,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,63,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,63,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,127,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,255,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,511,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,511,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,1023,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,1023,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,2047,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,2047,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,4095,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,4095,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,8191,0.019600000232458115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,8191,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,float16,16383,0.02532800038655599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,1,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,64,0,1,float16,fp8,16383,0.023999998966852825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,3,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,1,0.01249066616098086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,3,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,7,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,15,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,31,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,31,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,63,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,127,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,255,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,511,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,511,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,1023,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,1023,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,2047,0.016890666137139004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,2047,0.015861333658297855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,4095,0.02124800036350886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,4095,0.019621333728233974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,8191,0.03346666693687439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,8191,0.029178666571776073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,float16,16383,0.056133334835370384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,1,0.027679999669392902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,64,0,1,float16,fp8,16383,0.04626133541266123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,1,0.027610667049884796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,3,0.027317332724730175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,3,0.027632000545660656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,7,0.02775999903678894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,7,0.02735999971628189
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,15,0.027429332335789997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,15,0.02773866554101308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,31,0.027664000789324444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,31,0.027514666318893433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,63,0.027589333554108936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,63,0.027237333357334137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,127,0.027429332335789997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,127,0.027535999814669292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,255,0.02741866558790207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,255,0.027503999571005504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,511,0.04234133164087931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,511,0.03759466608365377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,1023,0.06465599934260051
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,1023,0.05813866853713989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,float16,2047,0.11392533779144287
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,1,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,64,0,1,float16,fp8,2047,0.09826667110125224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,1,0.012703999876976013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,3,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,3,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,7,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,7,0.011786667009194693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,15,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,15,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,31,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,31,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,63,0.012650666137536367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,63,0.012229333321253458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,127,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,127,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,255,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,511,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,511,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,1023,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,1023,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,2047,0.023205332458019257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,2047,0.022858666876951855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,4095,0.03364266703526179
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,4095,0.029530666768550873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,8191,0.05857066810131073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,8191,0.0479360024134318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,float16,16383,0.10171199838320415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,1,0.04590400060017904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,64,0,1,float16,fp8,16383,0.0862666666507721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,1,0.04597333570321401
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,3,0.0462773342927297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,3,0.045696000258127846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,7,0.04587199787298838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,7,0.04594666759173075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,15,0.04566933214664459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,15,0.04566933214664459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,31,0.045706664522488914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,31,0.043893332282702126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,63,0.04594666759173075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,63,0.04387199878692627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,127,0.045834665497144066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,127,0.04572266836961111
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,255,0.04794133206208547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,255,0.04359999795754751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,511,0.07238399982452393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,511,0.06633066634337108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,float16,1023,0.11617599924405415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,64,0,1,float16,fp8,1023,0.10426132877667744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,1,0.08143466711044312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,3,0.08261866867542267
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,1,0.07860266665617625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,3,0.07900799810886383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,7,0.08275733391443889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,7,0.07892799874146779
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,15,0.08197866876920064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,15,0.07877866427103679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,31,0.08157866696516673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,31,0.07914666831493378
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,63,0.08109333117802937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,63,0.07920533418655396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,127,0.08269333342711131
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,127,0.07854400078455608
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,float16,255,0.08292266726493835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,64,0,1,float16,fp8,255,0.07932800054550171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,1,0.15499732891718546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,1,0.14864533146222433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,3,0.15481066703796387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,3,0.148799995581309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,7,0.1553973356882731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,7,0.14869866768519083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,15,0.15500266353289285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,15,0.14878400166829428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,31,0.15521066387494406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,31,0.14841066797574362
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,63,0.15476800004641214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,63,0.14830933014551798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,1,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,float16,127,0.15446933110555014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,1,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,3,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,3,0.013754667093356451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,64,0,1,float16,fp8,127,0.15030399958292642
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,7,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,7,0.013647999614477158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,15,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,15,0.01370666672786077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,31,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,31,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,63,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,127,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,63,0.013482666263977686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,127,0.013610667238632837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,255,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,255,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,511,0.015770666301250458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,511,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,1023,0.019930666933457058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,1023,0.018981333822011948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,2047,0.029930666089057922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,2047,0.027797333896160126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,4095,0.050101334849993386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,4095,0.040405333042144775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,8191,0.0812853326400121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,8191,0.06881066660086314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,float16,16383,0.14525866508483887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,64,0,1,float16,fp8,16383,0.1218933363755544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,1,0.29631465673446655
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,1,0.28620266914367676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,3,0.2962239980697632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,3,0.28547734022140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,7,0.2970133423805237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,7,0.2860213319460551
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,15,0.29637332757314044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,15,0.2869173288345337
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,31,0.2977120081583659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,31,0.28643733263015747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,fp8,63,0.2857919931411743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,64,0,1,float16,float16,63,0.29703466097513836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,1,0.5804320176442465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,1,0.5596746603647867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,3,0.580069343249003
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,3,0.5587253173192342
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,7,0.5805439949035645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,7,0.5592853228251139
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,15,0.5793173313140869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,1,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,15,0.5582506656646729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,1,0.01533866673707962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,3,0.015386667102575302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,3,0.015279999623696009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,7,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,7,0.014767999450365702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,float16,31,0.5802880128224691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,15,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,15,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,64,0,1,float16,fp8,31,0.5596479972203573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,31,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,63,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,31,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,63,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,127,0.014922666052977243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,127,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,255,0.014885333677132925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,255,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,511,0.0194560003777345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,511,0.019253333409627277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,1023,0.025274666647116344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,1023,0.023599999646345775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,2047,0.04451199869314829
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,2047,0.03766400118668874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,4095,0.07069333394368489
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,4095,0.0625493327776591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,1,0.021354667842388153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,fp8,8191,0.10527466734250386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,64,0,1,float16,float16,8191,0.12408000230789185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,1,0.019215999792019527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,3,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,3,0.021018666525681812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,7,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,7,0.019519999623298645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,15,0.021402666966120403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,31,0.021082667013009388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,15,0.02120000123977661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,31,0.0195573332409064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,63,0.021194666624069214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,63,0.021002667645613354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,127,0.021685334543387096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,255,0.021359999974568684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,127,0.01913600042462349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,255,0.01926933353145917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,511,0.037434667348861694
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,511,0.02678400029738744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,1023,0.062208001812299095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,1023,0.039493332306543984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,2047,0.11341866850852966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,2047,0.06740266581376393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,1,0.011594666788975397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,float16,4095,0.21055465936660767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,40,40,128,0,1,float16,fp8,4095,0.11383466919263203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,1,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,3,0.011722666521867117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,3,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,7,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,7,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,15,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,15,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,31,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,31,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,63,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,63,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,127,0.011653333902359009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,127,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,255,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,255,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,511,0.013584000368913015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,511,0.012629333883523941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,1023,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,2047,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,1023,0.0136266661187013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,2047,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,4095,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,4095,0.016544000556071598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,8191,0.018874666343132656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,8191,0.021898667017618816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,float16,16383,0.04205333193143209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,1,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,40,40,128,0,1,float16,fp8,16383,0.025221332907676697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,1,0.011578666667143503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,3,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,7,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,7,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,15,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,15,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,31,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,63,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,63,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,127,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,255,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,511,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,511,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,1023,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,1023,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,2047,0.018895999838908512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,2047,0.017157333592573803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,4095,0.025493333737055462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,4095,0.02181333303451538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,8191,0.051856001218159996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,8191,0.0315786674618721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,float16,16383,0.08774933218955994
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,40,40,128,0,1,float16,fp8,16383,0.05403733253479004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,1,0.03225066761175791
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,3,0.03161066770553589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,1,0.02940266579389572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,3,0.029418667157491047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,7,0.03147733211517334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,7,0.029472000896930695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,15,0.03206400076548258
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,15,0.029487999776999157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,31,0.03215466688076655
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,31,0.03013866643110911
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,63,0.031701333820819855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,63,0.02993600070476532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,127,0.029317334294319153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,127,0.03183999905983607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,255,0.03810133288304011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,255,0.02940800040960312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,511,0.061893333991368614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,511,0.04172799984614054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,1023,0.10731200377146403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,1023,0.06316799918810527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,1,0.012682666381200155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,1,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,fp8,2047,0.11292266845703125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,3,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,40,40,128,0,1,float16,float16,2047,0.20503999789555868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,3,0.012149333953857422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,7,0.01191466674208641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,7,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,15,0.012063999970753988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,15,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,31,0.011823999385039011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,31,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,63,0.012671999633312225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,127,0.011829332758982977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,127,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,255,0.012469333906968435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,255,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,511,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,511,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,1023,0.018320000420014065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,1023,0.016832000265518825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,2047,0.027493332823117573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,2047,0.02359466751416524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,4095,0.05530133346716563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,4095,0.03338133295377096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,8191,0.0941546658674876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,8191,0.058448001742362976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,float16,16383,0.17055465777715048
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,1,0.05457599957784017
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,40,40,128,0,1,float16,fp8,16383,0.09967999656995137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,1,0.04810666541258494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,3,0.05446400245030721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,3,0.04794133206208547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,7,0.053930665055910744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,7,0.048021331429481506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,15,0.05538133283456167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,15,0.0462773342927297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,31,0.05410666763782501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,31,0.047610665361086525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,63,0.05411200225353241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,63,0.04977599779764811
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,127,0.056074668963750206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,127,0.04760533571243286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,255,0.06242666641871134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,255,0.0484799991051356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,511,0.10785599549611409
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,511,0.07254933317502339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,fp8,1023,0.11235733826955159
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,40,40,128,0,1,float16,float16,1023,0.19991999864578247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,1,0.10095999638239543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,1,0.08476799726486206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,3,0.09984532992045085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,3,0.08638933300971985
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,7,0.0990613301595052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,7,0.08489599823951721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,15,0.10115200281143188
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,15,0.08489599823951721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,31,0.09917866190274556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,31,0.08694932858149211
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,63,0.10035733381907146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,63,0.0853706697622935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,127,0.09924266735712688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,127,0.08683199683825175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,float16,255,0.11172266801198323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,40,40,128,0,1,float16,fp8,255,0.08654399712880452
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,1,0.18736000855763754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,1,0.1606826682885488
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,3,0.18655999501546225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,3,0.15898666779200235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,7,0.18755733966827393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,7,0.15967999895413718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,15,0.15957333644231161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,31,0.18717332681020102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,15,0.18818666537602743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,31,0.15933332840601602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,63,0.18555200099945068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,63,0.1602079967657725
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,1,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,fp8,127,0.15889066457748413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,1,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,3,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,3,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,7,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,40,40,128,0,1,float16,float16,127,0.18544000387191772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,7,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,15,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,15,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,31,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,31,0.013429333766301474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,63,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,63,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,127,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,127,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,255,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,255,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,511,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,511,0.015674666812022526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,1023,0.02149333308140437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,1023,0.01904533306757609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,2047,0.046949331959088646
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,2047,0.02959999938805898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,4095,0.048709332942962646
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,4095,0.07737066845099132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,8191,0.13804800311724344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,8191,0.08064533273379008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,fp8,16383,0.14166933298110962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,40,40,128,0,1,float16,float16,16383,0.25912533203760785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,1,0.3591839869817098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,1,0.30762133995691937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,3,0.3592373530069987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,3,0.30802132685979206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,7,0.35884801546732586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,7,0.3081173300743103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,15,0.3590879837671916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,15,0.3072693347930908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,31,0.3594079812367757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,31,0.30716800689697266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,fp8,63,0.30749332904815674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,40,40,128,0,1,float16,float16,63,0.35968534151713055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,1,0.7053866386413574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,1,0.5994346539179484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,3,0.7060373624165853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,3,0.598682681719462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,7,0.7059093316396078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,7,0.5995626846949259
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,15,0.5989280144373575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,15,0.7057813008626302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,1,0.017050666113694508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,float16,31,0.7060533364613851
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,1,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,3,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,3,0.015370666980743408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,7,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,7,0.017685333887736004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,15,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,15,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,31,0.01785600061217944
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,31,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,63,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,127,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,63,0.015957333147525787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,127,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,255,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,255,0.015610666324694952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,511,0.021984001000722248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,511,0.018976000448067982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,40,40,128,0,1,float16,fp8,31,0.5990399916966757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,1023,0.025381334125995636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,1023,0.03978666663169861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,2047,0.06858133276303609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,2047,0.04385066529115041
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,4095,0.12193066875139873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,4095,0.07063999772071838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,1,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,1,0.01747200017174085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,float16,8191,0.22563199202219644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,40,40,128,0,1,float16,fp8,8191,0.12198932965596516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,3,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,3,0.017653333644072216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,7,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,7,0.017344000438849132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,15,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,15,0.017690667261679966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,31,0.017344000438849132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,31,0.017450666675964992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,63,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,63,0.017610666652520496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,127,0.016997333616018295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,127,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,255,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,255,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,511,0.023306667804718018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,511,0.021573332448800404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,1023,0.03376533339420954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,1023,0.029663999875386555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,2047,0.056176001826922096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,2047,0.05005866785844167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,4095,0.09417066971460979
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,4095,0.0806933343410492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,1,0.01062400018175443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,fp8,8191,0.14173332850138345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,64,0,1,float16,float16,8191,0.16774932543436685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,7,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,7,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,15,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,31,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,31,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,63,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,127,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,127,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,255,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,255,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,511,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,511,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,1023,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,1023,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,2047,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,2047,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,4095,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,4095,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,8191,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,8191,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,16383,0.021242665747801464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,16383,0.020714666694402695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,1,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,3,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,float16,32767,0.03367999941110611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,64,0,1,float16,fp8,32767,0.02914133419593175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,7,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,7,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,15,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,15,0.010608000059922537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,31,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,63,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,127,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,255,0.010565333068370819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,255,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,511,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,511,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,1023,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,1023,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,2047,0.015344000111023584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,2047,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,4095,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,4095,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,8191,0.023376000424226124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,8191,0.021381333470344543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,16383,0.03555733213822047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,16383,0.03012266755104065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,1,0.02517866591612498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,float16,32767,0.05639466643333435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,1,0.025610665480295818
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,64,0,1,float16,fp8,32767,0.05009600023428599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,3,0.02385599911212921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,3,0.02518400053183238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,7,0.025311999022960663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,7,0.02418133368094762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,15,0.024080000817775726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,15,0.025311999022960663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,31,0.02351466566324234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,31,0.023647998770078022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,63,0.02532266577084859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,63,0.023242667317390442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,127,0.023376000424226124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,127,0.023311999936898548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,255,0.025274666647116344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,255,0.023599999646345775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,511,0.03737066686153412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,511,0.03357866654793421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,1023,0.055871998270352684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,1023,0.04978133241335551
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,2047,0.09690666198730469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,2047,0.0848426620165507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,1,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,float16,4095,0.16966400543848673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,64,0,1,float16,fp8,4095,0.14406399925549826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,3,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,3,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,7,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,15,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,15,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,31,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,63,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,63,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,127,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,255,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,255,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,511,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,511,0.01259200026591619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,1023,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,1023,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,2047,0.018687999496857326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,2047,0.01695999999841054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,4095,0.02325333406527837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,4095,0.021583999196688335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,8191,0.03772799919048945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,8191,0.02940800040960312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,16383,0.05824000140031179
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,16383,0.05208533505598704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,1,0.03749866783618927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,float16,32767,0.10037333766619365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,1,0.037674665451049805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,64,0,1,float16,fp8,32767,0.08593599994977315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,3,0.03935466706752777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,7,0.0374293327331543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,3,0.03863999992609024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,7,0.03756800045569738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,15,0.03955200066169103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,15,0.037946666280428566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,31,0.03825066735347112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,63,0.03982933362325033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,31,0.038560000558694206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,63,0.03733866661787033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,127,0.037674665451049805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,127,0.037632000943024956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,255,0.03765333443880081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,255,0.04251733422279358
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,511,0.060005332032839455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,511,0.056218668818473816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,1023,0.0956106682618459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,1023,0.08583999673525493
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,float16,2047,0.17004799842834473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,64,0,1,float16,fp8,2047,0.14813333749771118
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,1,0.06635199983914693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,1,0.06471999982992808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,3,0.06646400193373363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,3,0.06444799900054932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,7,0.06612266600131989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,7,0.06452266871929169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,15,0.06659199794133504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,15,0.06435733536879222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,31,0.06651733318964641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,31,0.06451733410358429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,63,0.06633066634337108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,63,0.06418666740258534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,127,0.06805333495140076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,127,0.06400533517201741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,255,0.06975466509660085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,255,0.06612800061702728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,float16,511,0.10647466778755188
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,64,0,1,float16,fp8,511,0.09727467099825542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,1,0.1255573332309723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,1,0.11970133582750957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,3,0.1256053348382314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,3,0.11983999609947205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,7,0.12557333707809448
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,7,0.11934399604797363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,15,0.12557866175969443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,15,0.11964799960454305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,31,0.12589333454767862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,31,0.11960533261299133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,63,0.12573867042859396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,63,0.11990400155385335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,127,0.12585600217183432
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,127,0.1216373344262441
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,1,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,float16,255,0.12626133362452188
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,1,0.012741333494583765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,3,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,64,0,1,float16,fp8,255,0.11975466211636861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,3,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,7,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,7,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,15,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,31,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,31,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,63,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,63,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,127,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,127,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,255,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,511,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,511,0.013562666873137156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,1023,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,255,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,1023,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,2047,0.023183998962243397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,2047,0.02143466720978419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,4095,0.03941866755485535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,4095,0.03123733401298523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,8191,0.060309335589408875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,8191,0.05221333106358846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,16383,0.0867733359336853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,16383,0.10259733597437541
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,fp8,32767,0.15450132886568704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,64,0,1,float16,float16,32767,0.18435200055440268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,1,0.24050666888554892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,1,0.23076266050338745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,3,0.23035732905069986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,3,0.24042133490244547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,7,0.2405760089556376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,7,0.2306399941444397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,15,0.2323413292566935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,15,0.24051199356714884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,31,0.2402613361676534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,31,0.2307200034459432
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,63,0.24036266406377158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,63,0.2320906718571981
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,float16,127,0.2383413314819336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,64,0,1,float16,fp8,127,0.23214399814605713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,1,0.4672906796137492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,1,0.4508533477783203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,3,0.46803200244903564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,3,0.4495306809743245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,7,0.4668000141779582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,7,0.44947199026743573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,15,0.46821868419647217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,15,0.44966399669647217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,31,0.4498080015182495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,31,0.46718398729960126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,1,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,1,0.014490666488806406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,3,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,3,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,7,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,7,0.013994666437307993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,15,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,15,0.01422400027513504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,float16,63,0.46795201301574707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,31,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,31,0.014149333039919535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,63,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,64,0,1,float16,fp8,63,0.4496266841888428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,63,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,127,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,127,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,255,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,255,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,511,0.017136000096797943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,511,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,1023,0.023205332458019257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,1023,0.021231998999913532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,2047,0.040287998815377556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,2047,0.03154666721820831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,4095,0.06080000102519989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,4095,0.05392000079154968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,8191,0.1035146713256836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,8191,0.08687999844551086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,1,0.01916266605257988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,float16,16383,0.18838399648666382
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,64,0,1,float16,fp8,16383,0.15636266271273294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,1,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,3,0.019354666272799175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,3,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,7,0.01958400011062622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,15,0.018826667219400406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,7,0.017551999539136887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,31,0.01877333347996076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,15,0.018751999984184902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,31,0.017504000415404636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,63,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,63,0.01886933296918869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,127,0.020106667031844456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,127,0.018735999862353008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,255,0.019071999937295914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,255,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,511,0.03387200087308884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,511,0.022778667509555817
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,1023,0.05180266499519348
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,1023,0.03446399917205175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,2047,0.09339732925097148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,2047,0.05630399783452352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,4095,0.16958399613698324
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,4095,0.0932426651318868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,1,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,fp8,8191,0.16507200400034586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,32,128,0,1,float16,float16,8191,0.3206506570180257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,1,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,7,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,3,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,15,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,15,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,31,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,31,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,63,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,255,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,255,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,511,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,511,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,1023,0.01201066623131434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,1023,0.012645332763592402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,2047,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,2047,0.01340266689658165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,4095,0.014794666320085526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,4095,0.016917333006858826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,8191,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,8191,0.017194667210181553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,16383,0.03444266567627589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,16383,0.021354667842388153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,1,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,float16,32767,0.05478399991989136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,32,128,0,1,float16,fp8,32767,0.03376533339420954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,3,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,3,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,7,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,15,0.010687999427318573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,31,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,63,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,127,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,255,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,255,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,511,0.012666666259368261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,511,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,1023,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,1023,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,2047,0.016677333662907284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,2047,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,4095,0.019274666905403137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,4095,0.018581333259741466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,8191,0.023237332701683044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,8191,0.03581333408753077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,16383,0.056330665946006775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,16383,0.03648533423741659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,1,0.027434666951497395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,fp8,32767,0.056015998125076294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,32,128,0,1,float16,float16,32767,0.09798933068911235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,1,0.02515200028816859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,3,0.027461332579453785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,3,0.02550400048494339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,7,0.027280000348885853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,7,0.02517866591612498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,15,0.02759466568628947
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,15,0.02499199906984965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,31,0.02741333345572154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,31,0.02531733363866806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,63,0.027327999472618103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,63,0.025290665527184803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,127,0.027248000105222065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,127,0.02510933329661687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,255,0.032127998769283295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,255,0.02514133354028066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,511,0.051685333251953125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,511,0.036746665835380554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,1023,0.08935466408729553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,1023,0.05392533540725708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,2047,0.16880534092585245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,2047,0.09531733393669128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,1,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,float16,4095,0.31885866324106854
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,1,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,32,128,0,1,float16,fp8,4095,0.16710933049519858
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,3,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,3,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,7,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,7,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,15,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,31,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,15,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,63,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,127,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,127,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,255,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,255,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,511,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,511,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,1023,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,1023,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,2047,0.02090666691462199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,2047,0.01913600042462349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,4095,0.03762666632731756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,4095,0.023317334552605946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,8191,0.05818133552869161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,8191,0.03758399933576584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,16383,0.0995146632194519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,16383,0.0574239989121755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,1,0.04580266773700714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,float16,32767,0.1808906594912211
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,1,0.03973866750796636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,32,128,0,1,float16,fp8,32767,0.09611200292905171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,3,0.04529066880544027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,3,0.03961600114901861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,7,0.0452106644709905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,7,0.03969600051641464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,15,0.045968001087506614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,15,0.039605334401130676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,31,0.045834665497144066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,31,0.04012800008058548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,63,0.04465599854787191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,63,0.03966933240493139
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,127,0.03968533376852671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,255,0.052330667773882546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,255,0.04171733558177948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,127,0.048432002464930214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,511,0.08899733424186707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,511,0.060191998879114784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,1023,0.1639893352985382
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,1023,0.09402133027712505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,fp8,2047,0.16850666205088297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,32,128,0,1,float16,float16,2047,0.3158239920934041
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,1,0.08277333279450734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,1,0.0687306672334671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,3,0.08303466439247131
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,3,0.07007466753323872
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,7,0.08092266817887624
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,7,0.06915733218193054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,15,0.08270399769147237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,15,0.07043733199437459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,31,0.08072000245253245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,31,0.06969599922498067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,63,0.08250666658083598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,63,0.06849066913127899
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,127,0.08080533146858215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,255,0.07072533170382182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,255,0.09071466326713562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,127,0.07052800059318542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,float16,511,0.16447466611862183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,32,128,0,1,float16,fp8,511,0.10638399918874104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,1,0.15080533425013223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,1,0.13000532984733582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,3,0.15064000089963278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,3,0.12991467118263245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,7,0.1515733301639557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,7,0.13136000434557596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,15,0.15058666467666626
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,15,0.13053866227467856
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,31,0.15181333820025125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,31,0.13037866353988647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,63,0.1300320029258728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,63,0.15262400110562643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,127,0.1511253317197164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,127,0.13004266222318014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,float16,255,0.17004267374674478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,1,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,1,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,32,128,0,1,float16,fp8,255,0.13089600205421448
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,3,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,3,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,7,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,7,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,15,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,15,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,31,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,31,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,63,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,63,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,127,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,127,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,255,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,511,0.01479999969402949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,511,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,1023,0.019013332823912304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,1023,0.016885332763195038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,2047,0.03760000069936117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,2047,0.023376000424226124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,4095,0.058287998040517174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,4095,0.037818667789300285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,8191,0.10070400436719258
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,8191,0.059402664502461754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,16383,0.1823093295097351
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,16383,0.0997973382472992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,float16,32767,0.34755198160807294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,32,128,0,1,float16,fp8,32767,0.18025066455205283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,1,0.29047467311223346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,1,0.24846400817235312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,3,0.29075199365615845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,3,0.24950400988260904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,7,0.2903253237406413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,15,0.29022934039433795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,15,0.24874667326609293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,7,0.24905065695444742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,31,0.29148799180984497
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,31,0.248799999554952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,63,0.28991466760635376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,63,0.24914133548736572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,float16,127,0.2887679934501648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,32,128,0,1,float16,fp8,127,0.2487199902534485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,1,0.5664106607437134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,1,0.48239465554555255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,3,0.566431999206543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,3,0.4822400013605754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,7,0.5660053491592407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,7,0.48259198665618896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,15,0.5665546655654907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,15,0.4821919997533162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,31,0.5665333271026611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,31,0.48340801397959393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,1,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,1,0.017653333644072216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,3,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,3,0.01676799977819125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,float16,63,0.566319982210795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,7,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,32,128,0,1,float16,fp8,63,0.4825119972229004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,7,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,15,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,15,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,31,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,31,0.014826666563749313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,63,0.015477333217859268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,63,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,127,0.014720000326633453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,127,0.014837333311637243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,255,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,255,0.01482133318980535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,511,0.018911999960740406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,511,0.017050666113694508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,1023,0.033914667864640556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,1023,0.021322667598724365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,2047,0.05916266640027364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,2047,0.03953066716591517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,4095,0.06019733349482218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,4095,0.10089600086212158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,8191,0.10110400120417277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,8191,0.1841599941253662
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,1,0.01658133293191592
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,1,0.017008000363906223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,3,0.01695999999841054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,fp8,16383,0.1830986738204956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,3,0.017125333348910015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,32,128,0,1,float16,float16,16383,0.34914131959279376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,7,0.016986666868130367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,7,0.01746133342385292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,15,0.016805333395799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,15,0.016976000120242436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,31,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,31,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,63,0.016976000120242436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,63,0.016890666137139004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,127,0.01752000053723653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,127,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,255,0.01747200017174085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,255,0.017050666113694508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,511,0.02139200021823247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,511,0.01926933353145917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,1023,0.029978667696317036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,1023,0.0272533322374026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,2047,0.04996266464392344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,2047,0.04389866689840952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,4095,0.07069333394368489
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,4095,0.0823466678460439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,1,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,float16,8191,0.14545599619547525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,1,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,64,0,1,float16,fp8,8191,0.12201066811879475
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,3,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,7,0.010608000059922537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,7,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,15,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,31,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,63,0.010527999450763067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,63,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,127,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,127,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,255,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,511,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,511,0.012682666381200155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,1023,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,1023,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,2047,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,2047,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,4095,0.015546667079130808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,4095,0.015471999843915304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,8191,0.017375999440749485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,8191,0.01720000058412552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,16383,0.020901332298914593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,16383,0.019466667125622433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,1,0.010687999427318573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,float16,32767,0.029205332199732464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,64,0,1,float16,fp8,32767,0.025439999997615814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,3,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,3,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,15,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,63,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,63,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,127,0.012362666428089142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,255,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,255,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,511,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,511,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,1023,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,1023,0.014789332946141561
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,2047,0.015770666301250458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,2047,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,4095,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,4095,0.01754133279124896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,8191,0.02022933339079221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,8191,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,16383,0.030058667063713074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,16383,0.025290665527184803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,1,0.021082667013009388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,1,0.021253332495689392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,float16,32767,0.044293334086736046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,64,0,1,float16,fp8,32767,0.03878933439652125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,3,0.021253332495689392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,7,0.021482666333516438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,3,0.021770666042963665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,7,0.02102400114138921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,15,0.02144533395767212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,15,0.021290667355060577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,31,0.02186133215824763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,31,0.021322667598724365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,63,0.02141333371400833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,63,0.021162666380405426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,127,0.021733333667119343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,127,0.021920000513394673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,255,0.021589333812395733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,255,0.021205333371957142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,511,0.03108799954255422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,1023,0.04558399816354116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,511,0.02824000020821889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,1023,0.04161066561937332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,2047,0.07904533545176189
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,2047,0.07016533116499583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,float16,4095,0.136245330174764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,1,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,64,0,1,float16,fp8,4095,0.11756799618403117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,3,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,7,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,7,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,15,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,15,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,31,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,63,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,127,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,127,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,255,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,511,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,511,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,1023,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,1023,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,2047,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,2047,0.01684800038735072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,4095,0.023445333043734234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,4095,0.02141333371400833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,8191,0.036101333796978
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,8191,0.029824001093705494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,16383,0.058304001887639366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,16383,0.049866666396458946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,1,0.03150933235883713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,float16,32767,0.09975999593734741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,64,0,1,float16,fp8,32767,0.08692800005276997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,1,0.031354665756225586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,3,0.031744000812371574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,3,0.03165333221356074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,7,0.03141333411137263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,7,0.03186666717131933
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,15,0.03148266673088074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,15,0.03194133440653483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,31,0.03160000095764796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,31,0.031504000226656594
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,63,0.0321066677570343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,63,0.031386665999889374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,127,0.03173866619666418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,127,0.031727999448776245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,255,0.0335359995563825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,255,0.030810666580994923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,511,0.04990933338801066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,511,0.0444106658299764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,1023,0.07690666615962982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,1023,0.06860266625881195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,float16,2047,0.13689600427945456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,64,0,1,float16,fp8,2047,0.11770133177439372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,1,0.05379733443260193
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,1,0.052015999952952065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,3,0.053818667928377785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,3,0.051856001218159996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,7,0.05378133555253347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,7,0.051925331354141235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,15,0.05189866820971171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,31,0.05382933219273885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,31,0.051781331499417625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,63,0.05221333106358846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,15,0.05275199810663859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,63,0.052111998200416565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,127,0.053914666175842285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,127,0.05203733344872793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,255,0.056736002365748085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,255,0.05211733281612396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,fp8,511,0.0766186664501826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,64,0,1,float16,float16,511,0.08295466502507527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,1,0.09316266576449077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,1,0.09749866525332133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,3,0.09114133318265279
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,3,0.09717333316802979
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,7,0.09735467036565144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,15,0.0909546713034312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,7,0.09351467092831929
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,15,0.09719467163085938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,31,0.09735467036565144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,31,0.09098133444786072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,63,0.09308266639709473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,63,0.09758399923642476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,127,0.09742400050163269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,127,0.09300800164540608
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,1,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,float16,255,0.10122666756312053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,1,0.011653333902359009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,3,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,64,0,1,float16,fp8,255,0.09337066610654195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,3,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,7,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,7,0.012714666624863943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,15,0.012448000411192576
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,15,0.012671999633312225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,31,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,31,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,63,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,127,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,63,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,127,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,255,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,255,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,511,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,511,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,1023,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,1023,0.01534933348496755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,2047,0.023071999351183575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,2047,0.021530665457248688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,4095,0.035775999228159584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,4095,0.03002133220434189
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,8191,0.05933333436648051
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,8191,0.04971200227737427
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,16383,0.10141866405804952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,16383,0.08681066830952962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,float16,32767,0.18541866540908813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,64,0,1,float16,fp8,32767,0.1551466683546702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,1,0.18357867002487183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,1,0.1774186690648397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,3,0.1851253310839335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,3,0.17683732509613037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,7,0.18422400951385498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,7,0.17706666390101114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,15,0.18434133132298788
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,15,0.17725332578023276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,31,0.1850666602452596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,31,0.17697066068649292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,63,0.18495466311772665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,63,0.1771999994913737
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,float16,127,0.18555732568105063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,64,0,1,float16,fp8,127,0.17931199073791504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,1,0.355461319287618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,1,0.3415786822636922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,3,0.35582931836446124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,3,0.34246933460235596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,7,0.3563520113627116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,7,0.3414880037307739
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,15,0.3559253215789795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,15,0.3410559892654419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,31,0.3552693525950114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,31,0.3414880037307739
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,1,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,1,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,3,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,float16,63,0.35547733306884766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,3,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,7,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,7,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,15,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,15,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,31,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,31,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,63,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,63,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,127,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,127,0.013365333278973898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,255,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,255,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,511,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,511,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,1023,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,1023,0.018885333091020584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,2047,0.03153600047032038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,2047,0.027669332921504974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,64,0,1,float16,fp8,63,0.3412906726201375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,4095,0.05007466673851013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,4095,0.041840001940727234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,8191,0.08072533210118611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,8191,0.07023466626803081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,1,0.017103999853134155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,1,0.016901332885026932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,float16,16383,0.1444000005722046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,3,0.017157333592573803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,64,0,1,float16,fp8,16383,0.12200533350308736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,3,0.01695466662446658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,7,0.016997333616018295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,7,0.01695999999841054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,15,0.016970666746298473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,15,0.016837333639462788
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,31,0.01695466662446658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,63,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,31,0.01691199963291486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,63,0.01681600014368693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,127,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,127,0.016986666868130367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,255,0.017103999853134155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,255,0.016095999628305435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,511,0.02513599892457326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,511,0.019978666057189304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,1023,0.045461331804593406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,1023,0.029296000798543293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,2047,0.08028266827265422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,2047,0.0513973335425059
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,4095,0.08101333181063335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,4095,0.14256532986958823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,1,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,fp8,8191,0.1442453364531199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,1,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,24,24,128,0,1,float16,float16,8191,0.2691466609636943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,3,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,3,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,7,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,7,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,31,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,63,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,63,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,127,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,127,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,255,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,255,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,511,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,511,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,1023,0.012736000120639801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,2047,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,2047,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,4095,0.01693333312869072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,4095,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,8191,0.019493332753578823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,8191,0.017114666601022083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,16383,0.026399999856948853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,16383,0.021007999777793884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,float16,32767,0.04577599962552389
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,24,24,128,0,1,float16,fp8,32767,0.027664000789324444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,3,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,7,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,7,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,15,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,15,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,31,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,63,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,127,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,127,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,255,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,511,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,511,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,1023,0.014165333161751429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,1023,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,2047,0.016997333616018295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,2047,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,4095,0.0191040001809597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,4095,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,8191,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,8191,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,16383,0.04649066428343455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,16383,0.029264000554879505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,1,0.023472001155217487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,float16,32767,0.07518399755160014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,24,24,128,0,1,float16,fp8,32767,0.04515199859937032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,1,0.021055998901526134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,3,0.02317333221435547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,3,0.022570667167504627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,7,0.0233599990606308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,7,0.021274665991465252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,15,0.023386667172114056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,15,0.0229120006163915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,31,0.02333866556485494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,31,0.02310933421055476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,63,0.023232000569502514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,63,0.021045332153638203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,127,0.023050665855407715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,127,0.021205333371957142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,255,0.025125332176685333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,255,0.02128533273935318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,511,0.03123733401298523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,511,0.04384533564249674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,1023,0.07291199763615926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,2047,0.07867733140786488
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,2047,0.13402666648228964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,1023,0.04612799982229868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,1,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,1,0.011973333855470022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,float16,4095,0.2515573302904765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,24,24,128,0,1,float16,fp8,4095,0.1344373325506846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,3,0.011701333026091257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,7,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,7,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,15,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,31,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,31,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,15,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,63,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,63,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,127,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,127,0.01163200040658315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,255,0.010527999450763067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,255,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,511,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,511,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,1023,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,1023,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,2047,0.019296000401178997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,2047,0.018751999984184902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,4095,0.03165333221356074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,4095,0.023050665855407715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,8191,0.05435200035572052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,8191,0.035749333600203194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,16383,0.09287466605504353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,16383,0.05618133147557577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,1,0.03565333286921183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,float16,32767,0.16665599743525186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,1,0.03285333265860876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,24,24,128,0,1,float16,fp8,32767,0.09602133433024089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,3,0.03579733272393545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,3,0.0330826664964358
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,7,0.03757333258787791
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,7,0.03368533402681351
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,15,0.03752533346414566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,15,0.033557333052158356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,31,0.035786665976047516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,31,0.03344533344109853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,63,0.03571200122435888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,63,0.033333333830038704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,127,0.03770666569471359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,127,0.032655999064445496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,255,0.04381866753101349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,255,0.03373866776625315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,511,0.07052800059318542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,511,0.04975466430187225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,1023,0.1288426617781321
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,1023,0.07453866799672444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,float16,2047,0.24482667446136475
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,1,0.06443733473618825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,24,24,128,0,1,float16,fp8,2047,0.13209600249926248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,1,0.05402133365472158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,3,0.0639519989490509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,3,0.05402666827042898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,7,0.06443733473618825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,7,0.05421866476535797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,15,0.06418666740258534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,15,0.05417599777380625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,31,0.06437333424886067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,31,0.053946668903032936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,63,0.06437333424886067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,63,0.054341331124305725
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,127,0.06424533327420552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,127,0.05611200133959452
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,255,0.07189866900444031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,255,0.05598933498064677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,float16,511,0.12746133406956991
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,1,0.10140800476074219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,1,0.11735999584197998
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,24,24,128,0,1,float16,fp8,511,0.08497599760691325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,3,0.11778666575749715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,7,0.11749866604804993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,3,0.10143466790517171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,7,0.10171733299891154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,15,0.11754133303960164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,15,0.10116266210873921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,31,0.11761066317558289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,63,0.1178559958934784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,31,0.1018986701965332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,63,0.101200004418691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,127,0.1174773375193278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,1,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,1,0.012154666086037954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,127,0.10318400462468465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,3,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,fp8,255,0.10331733028093974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,3,0.012778667112191519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,24,24,128,0,1,float16,float16,255,0.13175466656684875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,7,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,7,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,15,0.012650666137536367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,15,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,31,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,31,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,63,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,63,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,127,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,127,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,255,0.011594666788975397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,511,0.014426667243242264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,511,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,255,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,1023,0.01727466657757759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,1023,0.015610666324694952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,2047,0.02977066735426585
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,2047,0.023455999791622162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,4095,0.05619200070699056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,4095,0.03670933345953623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,8191,0.0957973301410675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,8191,0.0598826656738917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,16383,0.17281067371368408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,16383,0.09993599851926167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,fp8,32767,0.18125333388646445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,24,24,128,0,1,float16,float16,32767,0.33074132601420086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,1,0.22074133157730103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,1,0.19108267625172934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,3,0.22099200884501138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,3,0.1905919909477234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,7,0.2220319906870524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,7,0.19126933813095093
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,15,0.22037333250045776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,15,0.1897439956665039
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,31,0.2211573322614034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,31,0.1916960080464681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,63,0.22031466166178384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,63,0.18977065881093344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,float16,127,0.21988266706466675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,24,24,128,0,1,float16,fp8,127,0.19327465693155924
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,1,0.42846934000651044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,1,0.3696800072987874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,3,0.4288426637649536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,3,0.3680533170700073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,7,0.4283253351847331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,7,0.3688533306121826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,15,0.4288160006205241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,15,0.3680479923884074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,31,0.4286293188730876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,31,0.3690720001856486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,1,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,1,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,3,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,3,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,7,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,7,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,float16,63,0.42874133586883545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,15,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,15,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,31,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,31,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,63,0.01434133326013883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,63,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,127,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,127,0.013733333597580591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,255,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,255,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,511,0.016901332885026932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,511,0.016330666840076447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,1023,0.019093333433071773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,1023,0.024682665864626568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,2047,0.05006400247414907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,2047,0.03148799886306127
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,4095,0.0802346666653951
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,24,24,128,0,1,float16,fp8,63,0.36813334623972577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,4095,0.04991999765237173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,8191,0.14174399773279825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,8191,0.0809333324432373
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,1,0.014682666709025701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,1,0.01469333345691363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,fp8,16383,0.1423413356145223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,24,24,128,0,1,float16,float16,16383,0.2672106623649597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,3,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,3,0.014677333335081736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,7,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,15,0.014789332946141561
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,7,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,15,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,31,0.014720000326633453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,31,0.014954666296641031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,63,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,63,0.01351999988158544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,127,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,127,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,255,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,255,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,511,0.016714667280515034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,511,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,1023,0.021104000508785248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,1023,0.021269333859284718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,2047,0.040021332601706185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,2047,0.03142400085926056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,4095,0.05242133140563965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,4095,0.06026133398214976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,8191,0.10469866792360942
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,8191,0.08707200487454732
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,1,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,1,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,fp8,16383,0.1551093359788259
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,3,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,64,0,1,float16,float16,16383,0.18742932875951132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,3,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,7,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,15,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,31,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,63,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,127,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,127,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,511,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,511,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,1023,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,1023,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,2047,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,2047,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,4095,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,4095,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,8191,0.016869333883126576
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,8191,0.015594666202863058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,16383,0.019653332730134327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,16383,0.018933333456516266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,32767,0.023205332458019257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,32767,0.022197333474953968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,float16,65535,0.035360001027584076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,1,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,3,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,3,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,7,0.010602666685978571
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,7,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,15,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,64,0,1,float16,fp8,65535,0.0295413335164388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,31,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,15,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,31,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,63,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,63,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,127,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,255,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,255,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,511,0.011610666910807291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,511,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,1023,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,1023,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,2047,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,2047,0.013701333353916803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,4095,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,4095,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,8191,0.01714666684468587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,8191,0.01770666614174843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,16383,0.021359999974568684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,16383,0.023520000278949738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,32767,0.03573866685231527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,32767,0.029338667790095013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,1,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,1,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,float16,65535,0.05690133571624756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,3,0.01728533332546552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,7,0.017237332959969837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,64,0,1,float16,fp8,65535,0.05037866532802582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,3,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,7,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,15,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,15,0.01716800034046173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,31,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,31,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,63,0.01748266691962878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,63,0.0174346665541331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,127,0.017701332767804463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,127,0.018976000448067982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,255,0.017136000096797943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,255,0.017701332767804463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,511,0.021573332448800404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,511,0.021087999145189922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,1023,0.03400533397992452
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,1023,0.029258665939172108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,2047,0.056159997979799904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,2047,0.05041066805521647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,4095,0.09422933061917622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,4095,0.08276799817879994
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,1,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,float16,8191,0.16845866044362387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,1,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,3,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,64,0,1,float16,fp8,8191,0.14040000240008035
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,7,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,7,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,15,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,15,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,31,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,63,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,63,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,127,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,127,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,255,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,255,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,511,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,1023,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,1023,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,2047,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,2047,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,4095,0.019109333554903667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,4095,0.01858666663368543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,8191,0.023621333142121632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,8191,0.023152001202106476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,16383,0.037674665451049805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,16383,0.03143466760714849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,32767,0.058874666690826416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,32767,0.051781331499417625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,1,0.023381332556406658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,float16,65535,0.09945600231488545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,1,0.023370665808518726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,3,0.02456533412138621
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,64,0,1,float16,fp8,65535,0.08666666348775227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,7,0.024890666206677754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,3,0.02369066576162974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,7,0.02385066697994868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,15,0.02462399999300639
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,15,0.0236160010099411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,31,0.024447999894618988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,31,0.023455999791622162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,63,0.023445333043734234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,63,0.025029333929220837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,127,0.025498665869235992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,127,0.025434667865435284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,255,0.025311999022960663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,255,0.025786665578683216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,511,0.037477334340413414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,511,0.03339733431736628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,1023,0.05625600119431814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,1023,0.05060266455014547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,2047,0.09681600332260132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,2047,0.08558932940165202
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,fp8,4095,0.14455999930699667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,64,0,1,float16,float16,4095,0.17107200622558594
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,1,0.03991466760635376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,1,0.037818667789300285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,3,0.03861333429813385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,3,0.03828799972931544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,7,0.03941866755485535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,7,0.037808001041412354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,15,0.03984000037113825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,15,0.03775999943415324
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,31,0.039701332648595176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,31,0.037946666280428566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,63,0.037685332198937736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,63,0.0377813329299291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,127,0.03809600075085958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,127,0.03751466671625773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,255,0.041536000867684685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,511,0.060453335444132485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,511,0.05605333546797434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,255,0.038634667793909706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,float16,1023,0.095551997423172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,64,0,1,float16,fp8,1023,0.08531199892361958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,1,0.06683200101057689
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,1,0.06571733454863231
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,3,0.06710933148860931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,3,0.06489600241184235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,7,0.06666133304437001
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,7,0.06526933113733928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,15,0.0666293352842331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,15,0.06410133341948192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,31,0.06666133304437001
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,31,0.06463466584682465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,63,0.06640533109505971
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,63,0.06454400221506755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,127,0.06876799960931142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,127,0.06532266736030579
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,255,0.0703359991312027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,255,0.06638399759928386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,1,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,1,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,float16,511,0.10710933804512024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,3,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,3,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,7,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,7,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,64,0,1,float16,fp8,511,0.09704533219337463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,31,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,63,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,127,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,127,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,255,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,255,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,511,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,511,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,1023,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,1023,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,2047,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,2047,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,4095,0.023344000180562336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,4095,0.02128000060717265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,8191,0.03765333443880081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,8191,0.029701332251230877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,16383,0.05832533538341522
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,16383,0.05215999980767568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,32767,0.08501332998275757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,32767,0.09943999846776326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,fp8,65535,0.155130664507548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,64,0,1,float16,float16,65535,0.18342934052149454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,1,0.12592533230781555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,1,0.11939733227094014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,3,0.12607999642690024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,3,0.11983999609947205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,7,0.12572800119717917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,7,0.11957333485285442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,15,0.11981866757074992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,15,0.12564266721407572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,31,0.1260586678981781
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,31,0.11957866946856181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,63,0.12600533167521158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,63,0.11988266309102376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,127,0.12603200475374857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,127,0.12199999888737996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,float16,255,0.12821333607037863
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,64,0,1,float16,fp8,255,0.12032533685366313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,1,0.24106667439142862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,1,0.23220799366633096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,3,0.24239999055862427
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,3,0.2320586641629537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,7,0.2412853240966797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,7,0.23388266563415527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,15,0.23198399941126505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,31,0.24184532960255942
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,31,0.2327679991722107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,63,0.24140799045562744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,63,0.23043733835220337
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,1,0.0124746672809124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,1,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,3,0.011887999872366587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,3,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,7,0.012351999680201212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,7,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,15,0.011871999750534693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,127,0.23984533548355103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,fp8,127,0.2320479949315389
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,15,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,63,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,31,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,31,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,63,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,127,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,127,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,255,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,255,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,511,0.013658666362365087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,511,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,1023,0.015856000284353893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,1023,0.016554666062196095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,2047,0.0234400009115537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,2047,0.021344001094500225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,4095,0.039690665900707245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,4095,0.03166933357715607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,8191,0.0601440022389094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,8191,0.05243733525276184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,16383,0.10361599922180176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,16383,0.08702400326728821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,1,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,float16,32767,0.18573333819707236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,1,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,3,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,3,0.014725333700577417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,64,0,1,float16,fp8,32767,0.15457600355148315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,64,0,1,float16,float16,15,0.24132800102233887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,7,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,7,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,15,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,31,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,15,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,31,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,63,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,63,0.014794666320085526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,127,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,127,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,255,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,255,0.014725333700577417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,511,0.01887999971707662
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,511,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,1023,0.03420799970626831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,1023,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,2047,0.060415998101234436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,2047,0.04061333338419596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,4095,0.06036800146102905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,4095,0.10022399822870891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,8191,0.1832159956296285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,8191,0.10190932949384053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,1,0.01166933278242747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,1,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,3,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,fp8,16383,0.18366400400797525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,7,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,16,128,0,1,float16,float16,16383,0.3479520082473755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,7,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,15,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,31,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,63,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,63,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,127,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,255,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,511,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,511,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,1023,0.011898666620254517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,1023,0.012698666503032049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,2047,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,2047,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,4095,0.016901332885026932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,4095,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,8191,0.018735999862353008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,8191,0.017162666966517765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,16383,0.02165333429972331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,16383,0.019808000574509304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,32767,0.03717333326737086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,32767,0.02332799881696701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,1,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,1,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,3,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,float16,65535,0.055973331133524575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,7,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,16,128,0,1,float16,fp8,65535,0.03587199995915095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,15,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,15,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,63,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,127,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,127,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,255,0.010549332946538925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,255,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,511,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,511,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,1023,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,1023,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,2047,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,2047,0.014186666657527288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,4095,0.016176000237464905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,4095,0.015370666980743408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,8191,0.021173333128293354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,8191,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,16383,0.037087999284267426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,16383,0.023226665953795116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,32767,0.05630933245023092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,32767,0.035802667339642845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,1,0.019498666127522785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,1,0.018602666755517323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,float16,65535,0.09562666217486064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,3,0.019359999646743137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,16,128,0,1,float16,fp8,65535,0.05552533268928528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,3,0.016997333616018295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,7,0.019424000134070713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,7,0.017114666601022083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,15,0.018911999960740406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,15,0.01732800031701724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,31,0.01907733331123988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,31,0.01743999992807706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,63,0.01907733331123988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,63,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,127,0.01922133316596349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,255,0.019253333409627277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,127,0.019978666057189304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,255,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,511,0.0335359995563825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,511,0.023290666441122692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,1023,0.03380800038576126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,1023,0.053488001227378845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,2047,0.09479467074076335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,2047,0.05760000149408976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,4095,0.16915200153986612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,4095,0.09490133325258891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,1,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,fp8,8191,0.16711999972661337
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,16,128,0,1,float16,float16,8191,0.3202880024909973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,1,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,3,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,7,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,3,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,31,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,31,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,127,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,127,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,255,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,255,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,511,0.012671999633312225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,511,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,1023,0.013605333864688873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,1023,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,2047,0.017535999417304993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,2047,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,4095,0.018944000204404194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,4095,0.021850667893886566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,8191,0.04008000095685323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,8191,0.0234400009115537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,16383,0.05946666498978933
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,16383,0.03855466594298681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,32767,0.09917333722114563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,32767,0.058378666639328
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,1,0.027488000690937042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,1,0.025472000241279602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,float16,65535,0.18078400691350302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,3,0.027450665831565857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,7,0.027530667682488758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,3,0.025173333783944447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,16,128,0,1,float16,fp8,65535,0.09801066915194194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,7,0.025685332715511322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,15,0.027477333943049114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,15,0.025445332129796345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,31,0.027488000690937042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,31,0.026021334032217663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,63,0.02802666773398717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,63,0.025610665480295818
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,127,0.028010666370391846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,255,0.03275199979543686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,127,0.02553066611289978
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,255,0.025536000728607178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,511,0.05226133267084757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,511,0.03684266656637192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,1023,0.053904001911481224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,1023,0.09036266803741455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,2047,0.16948266824086508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,2047,0.09547733267148335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,1,0.04557866851488749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,float16,4095,0.31994134187698364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,16,128,0,1,float16,fp8,4095,0.168778657913208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,1,0.03989866624275843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,3,0.04664533336957296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,3,0.039887999494870506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,7,0.04584000011285146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,7,0.0397173340121905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,15,0.04582933088143667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,15,0.03979199876387914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,31,0.03942933430274328
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,31,0.04571733375390371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,63,0.045941332976023354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,63,0.03955733279387156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,127,0.04665066798528036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,127,0.039733332892258964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,255,0.04177066683769226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,511,0.0906826655069987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,255,0.053413331508636475
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,511,0.06074133515357971
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,float16,1023,0.1627253293991089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,16,128,0,1,float16,fp8,1023,0.09402666489283244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,1,0.0702453354994456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,1,0.08293866614500682
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,3,0.08082133531570435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,3,0.06850666801134746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,7,0.08270399769147237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,7,0.07018666466077168
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,15,0.08248533308506012
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,15,0.07020799815654755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,31,0.08241066833337148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,31,0.07049599786599477
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,63,0.08156799773375194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,63,0.07044266661008199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,127,0.08152000109354655
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,127,0.07115733126799266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,255,0.09113599856694539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,255,0.07236266632874806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,1,0.012639999389648438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,float16,511,0.16637333234151205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,3,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,7,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,16,128,0,1,float16,fp8,511,0.1074079970518748
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,7,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,15,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,31,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,127,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,127,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,255,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,255,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,511,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,511,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,1023,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,1023,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,2047,0.019199999670187633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,2047,0.01842133328318596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,4095,0.03786666691303253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,4095,0.023760000864664715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,8191,0.037632000943024956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,8191,0.05842133363087972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,16383,0.05728533367315928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,16383,0.09831466277440389
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,32767,0.0974079966545105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,32767,0.17855467398961386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,1,0.151146670182546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,float16,65535,0.3416373332341512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,16,128,0,1,float16,fp8,65535,0.17547200123469034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,1,0.1306880017121633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,3,0.15267733732859293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,3,0.13180800278981528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,7,0.15074132879575095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,7,0.1309333344300588
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,15,0.1525759994983673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,15,0.131632000207901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,31,0.15254933635393778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,31,0.13165866335233053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,63,0.1523413360118866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,63,0.13012267152468363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,127,0.15058666467666626
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,127,0.13192533453305563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,float16,255,0.1690559983253479
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,16,128,0,1,float16,fp8,255,0.13248533010482788
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,1,0.25085333983103436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,3,0.28991466760635376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,3,0.2500800093015035
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,1,0.29053332408269245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,7,0.29049066702524823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,7,0.2502773404121399
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,15,0.29050666093826294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,15,0.2487679918607076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,31,0.29105599721272785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,31,0.24927467107772827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,63,0.2916373411814372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,1,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,63,0.24968532721201578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,1,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,3,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,3,0.013616000612576803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,7,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,7,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,15,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,15,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,31,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,31,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,float16,127,0.288485328356425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,63,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,63,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,16,128,0,1,float16,fp8,127,0.24899200598398843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,127,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,127,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,255,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,511,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,511,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,1023,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,1023,0.016688000410795212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,2047,0.03807999938726425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,2047,0.02421333392461141
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,4095,0.05906666815280914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,4095,0.03925333420435587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,8191,0.10080533226331075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,8191,0.059290667374928795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,16383,0.10052800178527832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,16383,0.1820533275604248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,1,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,1,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,3,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,3,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,fp8,32767,0.18090667327245077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,7,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,7,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,16,128,0,1,float16,float16,32767,0.34758933385213214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,15,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,15,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,31,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,31,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,63,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,63,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,127,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,127,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,255,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,255,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,511,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,511,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,1023,0.019018666197856266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,1023,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,2047,0.03246400008598963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,2047,0.02749866743882497
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,4095,0.05042133231957754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,4095,0.04366933306058248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,8191,0.08309866487979889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,8191,0.07061866422494252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,1,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,3,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,fp8,16383,0.12197867035865784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,64,0,1,float16,float16,16383,0.14470932881037393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,7,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,15,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,15,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,31,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,63,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,127,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,255,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,255,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,511,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,511,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,1023,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,1023,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,2047,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,2047,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,4095,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,4095,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,8191,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,8191,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,16383,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,16383,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,32767,0.021482666333516438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,32767,0.021370666722456615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,1,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,1,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,3,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,float16,65535,0.029674666623274486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,7,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,64,0,1,float16,fp8,65535,0.025754667818546295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,15,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,7,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,31,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,15,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,31,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,63,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,127,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,127,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,255,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,255,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,511,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,511,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,1023,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,1023,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,2047,0.014896000425020853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,2047,0.015439999600251516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,4095,0.015402667224407196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,4095,0.01534933348496755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,8191,0.017621333400408428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,8191,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,16383,0.020960000654061634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,16383,0.019509332875410717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,32767,0.0315786674618721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,32767,0.025807999074459076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,1,0.015599999576807022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,float16,65535,0.04608533283074697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,1,0.01543466622630755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,64,0,1,float16,fp8,65535,0.03958933303753535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,3,0.01685333376129468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,3,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,7,0.016890666137139004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,15,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,7,0.016895999511082966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,15,0.016885332763195038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,31,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,31,0.016890666137139004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,63,0.017184000462293625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,127,0.016800000021855038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,63,0.01693333312869072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,127,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,255,0.01691199963291486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,255,0.016629333297411602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,511,0.01937599976857503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,511,0.019365333020687103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,1023,0.030415999392668407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,1023,0.02738133321205775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,2047,0.050160000721613564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,2047,0.04379733403523763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,4095,0.08328533172607422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,1,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,4095,0.07188799977302551
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,1,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,float16,8191,0.14587733149528503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,64,0,1,float16,fp8,8191,0.1220906674861908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,3,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,3,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,7,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,15,0.011701333026091257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,15,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,31,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,63,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,63,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,127,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,255,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,511,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,511,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,1023,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,1023,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,2047,0.01728533332546552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,2047,0.015418666104475657
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,4095,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,4095,0.01877333347996076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,8191,0.021290667355060577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,8191,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,16383,0.03368533402681351
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,16383,0.0273333340883255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,32767,0.04640000065167745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,32767,0.04208533465862274
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,1,0.021850667893886566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,1,0.021456000705560047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,float16,65535,0.07523199915885925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,3,0.021269333859284718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,64,0,1,float16,fp8,65535,0.06448000172773997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,3,0.021226666867733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,7,0.021007999777793884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,7,0.020997333029905956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,15,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,15,0.0207893339296182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,31,0.021114667256673176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,31,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,63,0.021194666624069214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,63,0.021189334491888683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,127,0.02163733293612798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,127,0.02107200026512146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,255,0.021168000996112823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,255,0.021295999487241108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,511,0.030037333567937214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,511,0.027061333258946735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,1023,0.04588800172011057
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,1023,0.040591999888420105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,2047,0.07896000146865845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,2047,0.07088533540566762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,float16,4095,0.13690132896105447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,64,0,1,float16,fp8,4095,0.1179146667321523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,1,0.03178133318821589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,1,0.03140799949566523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,3,0.031530665854612984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,3,0.031514666974544525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,7,0.031744000812371574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,7,0.03136000037193298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,15,0.0315786674618721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,15,0.03150933235883713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,31,0.03129599988460541
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,31,0.031530665854612984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,63,0.03146133323510488
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,63,0.03136533250411352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,127,0.031541332602500916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,127,0.03252800057331721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,255,0.03357866654793421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,255,0.029482667644818623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,511,0.05012266834576925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,float16,1023,0.07667199770609538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,1023,0.06824000179767609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,64,0,1,float16,fp8,511,0.04647466540336609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,1,0.05394133428732554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,1,0.05182399849096934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,3,0.052501335740089417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,7,0.0518506666024526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,3,0.052426666021347046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,7,0.051829333106676735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,15,0.05217599868774414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,15,0.051856001218159996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,31,0.054485330979029335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,31,0.05190399785836538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,63,0.05376533170541128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,63,0.051701332132021584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,127,0.05417599777380625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,127,0.05177066723505656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,255,0.05528533458709717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,255,0.05225066840648651
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,1,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,1,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,3,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,3,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,7,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,fp8,511,0.0765226682027181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,64,0,1,float16,float16,511,0.08320533235867818
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,31,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,31,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,63,0.01071999967098236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,63,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,127,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,127,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,511,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,511,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,1023,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,1023,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,2047,0.017071999609470367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,2047,0.01703466723362605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,4095,0.023434666295846302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,4095,0.021146667500336964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,8191,0.03534399966398875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,8191,0.029845332105954487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,16383,0.05816000203291575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,16383,0.05040533343950907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,32767,0.09947199622790019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,32767,0.0863200028737386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,float16,65535,0.1813919941584269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,1,0.09764800469080608
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,64,0,1,float16,fp8,65535,0.15466133753458658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,1,0.09288533528645833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,3,0.09758933385213216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,3,0.09156266848246257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,7,0.09292266766230266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,7,0.09759466846783955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,15,0.09768533706665039
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,15,0.09284800291061401
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,31,0.09785067041714986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,31,0.09172266721725464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,63,0.09710400303204854
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,63,0.09262933333714803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,127,0.0976106623808543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,127,0.09356799721717834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,float16,255,0.10016000270843506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,64,0,1,float16,fp8,255,0.09396800398826599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,1,0.1770026683807373
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,1,0.18581867218017578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,3,0.1848693291346232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,3,0.17757866779963175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,7,0.18445332845052084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,15,0.185370663801829
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,7,0.17753066619237265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,15,0.17780266205469766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,31,0.1851253310839335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,31,0.17733865976333618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,63,0.17724267641703287
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,63,0.18547733624776205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,1,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,1,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,3,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,3,0.012117333710193634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,7,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,7,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,15,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,15,0.01192533348997434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,31,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,31,0.011717333147923151
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,63,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,63,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,float16,127,0.18480533361434937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,127,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,127,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,255,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,255,0.01173866664369901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,64,0,1,float16,fp8,127,0.1779200037320455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,511,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,511,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,1023,0.01563199982047081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,1023,0.015530666957298914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,2047,0.023210667073726654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,2047,0.02310933421055476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,4095,0.03586666782697042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,4095,0.03141866624355316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,8191,0.06018666426340739
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,8191,0.05045333504676819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,16383,0.10160000125567119
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,16383,0.08650133013725281
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,1,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,1,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,3,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,3,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,float16,32767,0.18600000937779745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,64,0,1,float16,fp8,32767,0.15626666943232217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,7,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,7,0.013605333864688873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,15,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,15,0.013946666071812311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,31,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,31,0.0143306665122509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,63,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,63,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,127,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,127,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,255,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,255,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,511,0.016943999876578648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,511,0.016688000410795212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,1023,0.02306666721900304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,1023,0.018837332725524902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,2047,0.050026665131251015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,2047,0.03333866596221924
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,4095,0.08078933258851369
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,4095,0.04990399877230326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,8191,0.08273600041866302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,8191,0.1425226628780365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,1,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,1,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,fp8,16383,0.14331199725468954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,12,12,128,0,1,float16,float16,16383,0.26622400681177777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,7,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,15,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,31,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,31,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,63,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,127,0.010421333213647207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,127,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,255,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,511,0.012736000120639801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,511,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,1023,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,1023,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,2047,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,2047,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,4095,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,4095,0.015429332852363586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,8191,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,8191,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,16383,0.019365333020687103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,16383,0.01828266680240631
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,32767,0.025301332275072735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,32767,0.02124800036350886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,1,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,1,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,float16,65535,0.04601066807905833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,3,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,12,12,128,0,1,float16,fp8,65535,0.02943466603755951
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,7,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,7,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,15,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,31,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,31,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,63,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,63,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,127,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,127,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,255,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,511,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,511,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,1023,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,1023,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,2047,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,2047,0.016607999801635742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,4095,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,4095,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,8191,0.021040000021457672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,8191,0.017386666188637417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,16383,0.026288000245889027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,16383,0.021365332106749218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,32767,0.04621333380540212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,32767,0.031210665901501972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,1,0.01758933315674464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,float16,65535,0.07679466903209686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,1,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,3,0.01749333366751671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,3,0.017397332936525345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,7,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,12,12,128,0,1,float16,fp8,65535,0.046015997727712
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,7,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,15,0.017551999539136887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,15,0.016997333616018295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,31,0.01687466725707054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,31,0.01757866640885671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,63,0.017674667139848072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,63,0.01684800038735072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,127,0.017535999417304993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,127,0.017237332959969837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,255,0.01669866715868314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,255,0.01728533332546552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,511,0.021221332252025604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,511,0.026154667139053345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,1023,0.044010668992996216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,1023,0.029274667302767437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,2047,0.08044266700744629
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,2047,0.05177066723505656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,4095,0.14330666263898215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,4095,0.08157333234945933
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,3,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,3,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,float16,8191,0.2691359917322795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,7,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,12,12,128,0,1,float16,fp8,8191,0.1439626713593801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,15,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,15,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,31,0.010586666564146677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,31,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,63,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,127,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,127,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,255,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,255,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,511,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,511,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,1023,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,1023,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,2047,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,2047,0.016773333152135212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,4095,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,4095,0.01937599976857503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,8191,0.02752533306678136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,8191,0.02179733415444692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,16383,0.031888000667095184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,16383,0.048986668388048805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,32767,0.07870399951934814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,32767,0.04782933493455251
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,1,0.02334933231274287
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,1,0.023013333479563396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,float16,65535,0.13673599561055502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,3,0.023520000278949738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,3,0.022965334355831146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,12,12,128,0,1,float16,fp8,65535,0.07468800246715546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,7,0.023408000667889912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,7,0.023205332458019257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,15,0.023029332359631855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,15,0.02401600033044815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,31,0.023168000082174938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,31,0.023557332654794056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,63,0.023893333971500397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,63,0.02292266736427943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,127,0.02332266668478648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,127,0.021456000705560047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,255,0.025301332275072735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,255,0.021509334444999695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,511,0.04331199824810028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,511,0.031680000325044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,1023,0.07216000060240428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,1023,0.04422399898370107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,2047,0.13417067130406699
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,2047,0.07780799766381581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,float16,4095,0.2533973256746928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,12,12,128,0,1,float16,fp8,4095,0.13404800494511923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,1,0.03558400024970373
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,1,0.03332799921433131
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,3,0.035818666219711304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,3,0.03349333256483078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,7,0.03765333443880081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,7,0.033344000577926636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,15,0.03621866554021835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,15,0.03491200009981791
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,31,0.03730133424202601
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,31,0.033600000043710075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,63,0.035631999373435974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,127,0.03862933317820231
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,127,0.03331200033426285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,63,0.034373333056767784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,255,0.0422986646493276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,255,0.03523733218510946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,511,0.0710346649090449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,511,0.04964800179004669
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,float16,1023,0.12761599818865457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,12,12,128,0,1,float16,fp8,1023,0.07437333464622498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,1,0.06437333424886067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,3,0.06442666550477345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,1,0.05406933526198069
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,3,0.05414933462937673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,7,0.06465599934260051
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,15,0.06451199948787689
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,7,0.05426666637261709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,15,0.0543093333641688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,31,0.06446933249632518
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,31,0.05399466554323832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,63,0.06445333361625671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,63,0.05427733560403188
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,127,0.06462933123111725
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,127,0.05579733351866404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,255,0.07039466500282288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,1,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,255,0.05781333148479462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,1,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,3,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,float16,511,0.12727999687194824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,12,12,128,0,1,float16,fp8,511,0.082997332016627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,7,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,15,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,31,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,31,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,63,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,63,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,127,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,255,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,511,0.01268799975514412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,511,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,1023,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,1023,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,2047,0.01966399947802226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,2047,0.019317333896954853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,4095,0.030666666726271313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,4095,0.023605334262053173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,8191,0.05489066739877065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,8191,0.03655466685692469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,16383,0.09411199887593587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,16383,0.05635733405749003
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,32767,0.16733866930007935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,32767,0.09596266349156697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,float16,65535,0.315610667069753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,1,0.11839999755223592
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,12,12,128,0,1,float16,fp8,65535,0.17523733774820963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,1,0.1016533374786377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,3,0.11797866225242615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,3,0.10233599940935771
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,7,0.11789333820343018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,7,0.10187199711799622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,15,0.11766933401425679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,15,0.10154133041699727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,31,0.11785067121187846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,31,0.10299733281135559
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,63,0.11789333820343018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,63,0.1035093367099762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,127,0.11763200163841248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,127,0.10300800204277039
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,fp8,255,0.10439466436704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,12,12,128,0,1,float16,float16,255,0.13104533155759177
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,1,0.2205386757850647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,1,0.1901280085245768
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,3,0.22045334180196127
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,3,0.19051732619603476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,7,0.22060267130533853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,7,0.1908586621284485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,15,0.2206666668256124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,15,0.19001599152882895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,31,0.22216532627741495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,31,0.19148266315460205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,63,0.2210293412208557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,1,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,63,0.19012266397476196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,1,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,3,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,float16,127,0.2193546692530314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,7,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,7,0.01267733300725619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,12,12,128,0,1,float16,fp8,127,0.19132266441980997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,15,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,15,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,31,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,31,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,63,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,63,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,127,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,127,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,255,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,255,0.012778667112191519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,511,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,511,0.014837333311637243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,1023,0.016805333395799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,1023,0.01903466631968816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,2047,0.03150933235883713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,2047,0.023919999599456787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,4095,0.03647999962170919
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,4095,0.057333335280418396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,8191,0.09563199679056804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,8191,0.060565332571665444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,16383,0.09931199749310811
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,16383,0.17303999265034994
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,1,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,1,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,3,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,3,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,7,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,fp8,32767,0.18102933963139853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,12,12,128,0,1,float16,float16,32767,0.3283519943555196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,7,0.012709333250919977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,15,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,15,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,31,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,31,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,63,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,63,0.013712000101804733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,127,0.012538666526476542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,127,0.012709333250919977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,255,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,255,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,511,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,511,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,1023,0.015669333438078564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,1023,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,2047,0.02346133440732956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,2047,0.023200000325838726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,4095,0.04033066580692927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,4095,0.03145600110292435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,8191,0.06102400024731954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,8191,0.05376000205675761
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,16383,0.10351999600728352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,16383,0.08705066641171773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,1,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,1,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,3,0.010687999427318573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,fp8,32767,0.15686933199564615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,64,0,1,float16,float16,32767,0.1853546698888143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,3,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,7,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,15,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,7,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,15,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,31,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,63,0.010522666076819101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,127,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,127,0.010506667196750641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,255,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,255,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,511,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,511,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,1023,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,1023,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,2047,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,2047,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,4095,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,4095,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,8191,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,8191,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,16383,0.017488000293572743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,16383,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,32767,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,32767,0.018901333212852478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,65535,0.023552000522613525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,65535,0.021488000949223835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,1,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,3,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,3,0.011786667009194693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,7,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,float16,131071,0.039290666580200195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,7,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,15,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,15,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,31,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,64,0,1,float16,fp8,131071,0.03137599925200144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,31,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,63,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,127,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,255,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,511,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,511,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,1023,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,1023,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,2047,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,2047,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,4095,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,4095,0.016010666886965435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,8191,0.017008000363906223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,8191,0.017317333569129307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,16383,0.01916266605257988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,16383,0.01953599974513054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,32767,0.02345066765944163
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,32767,0.023285334308942158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,65535,0.0377866675456365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,65535,0.029818666477998097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,1,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,1,0.013904000322024027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,3,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,3,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,7,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,7,0.015344000111023584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,15,0.015477333217859268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,15,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,31,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,float16,131071,0.05746666590372721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,64,0,1,float16,fp8,131071,0.051125332713127136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,63,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,31,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,63,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,127,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,127,0.014746667196353277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,255,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,255,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,511,0.017018667111794155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,511,0.016885332763195038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,1023,0.023050665855407715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,1023,0.02124800036350886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,2047,0.04037333279848099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,2047,0.03194666653871536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,4095,0.06099733213583628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,4095,0.05366399884223938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,8191,0.10386666655540466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,8191,0.08701333403587341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,1,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,3,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,3,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,7,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,float16,16383,0.1876586675643921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,64,0,1,float16,fp8,16383,0.156741331020991
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,15,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,31,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,127,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,255,0.010506667196750641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,255,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,511,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,511,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,1023,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,1023,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,2047,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,2047,0.014762666076421738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,4095,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,4095,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,8191,0.01922133316596349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,8191,0.01716800034046173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,16383,0.023760000864664715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,16383,0.023232000569502514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,32767,0.0378560001651446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,32767,0.031093334158261616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,65535,0.05861333509286245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,65535,0.05205333232879639
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,1,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,1,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,3,0.01752000053723653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,float16,131071,0.10160000125567119
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,3,0.01800000046690305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,7,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,7,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,64,0,1,float16,fp8,131071,0.08939733107884724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,15,0.01699200024207433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,15,0.01727466657757759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,31,0.01716800034046173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,63,0.017157333592573803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,31,0.01706133286158244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,63,0.01700266698996226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,127,0.01710933322707812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,127,0.017386666188637417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,255,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,255,0.017344000438849132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,511,0.021375998854637146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,511,0.022218666970729828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,1023,0.03443199892838796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,1023,0.029653333127498627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,2047,0.05797333518664042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,2047,0.052560001611709595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,4095,0.09417600433031718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,4095,0.08159466584523518
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,1,0.023237332701683044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,1,0.025216000775496166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,float16,8191,0.1690666675567627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,3,0.02332266668478648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,3,0.025418666501839954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,7,0.025381334125995636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,64,0,1,float16,fp8,8191,0.14224533240000406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,7,0.02319466571013133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,15,0.025290665527184803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,15,0.025120000044504803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,31,0.023605334262053173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,63,0.02535466601451238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,31,0.025434667865435284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,63,0.02517866591612498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,127,0.02348800003528595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,127,0.023792001108328503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,255,0.025290665527184803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,255,0.025578667720158894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,511,0.03585066646337509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,511,0.03165333221356074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,1023,0.056074668963750206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,1023,0.05011733373006185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,float16,2047,0.09719467163085938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,64,0,1,float16,fp8,2047,0.084906667470932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,1,0.039733332892258964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,1,0.03787733366092046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,3,0.03762666632731756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,3,0.03737066686153412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,7,0.0376800000667572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,7,0.037808001041412354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,15,0.03786666691303253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,15,0.03844800094763438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,31,0.03766933331886927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,31,0.03731200098991394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,63,0.03752533346414566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,63,0.03938666731119156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,127,0.03755199909210205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,127,0.03750933210055033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,255,0.03770666569471359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,255,0.041519999504089355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,511,0.06018133461475372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,511,0.05603733162085215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,1,0.010656000425418219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,1,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,float16,1023,0.09498666723569234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,64,0,1,float16,fp8,1023,0.08509866396586101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,3,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,7,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,15,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,63,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,127,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,127,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,255,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,511,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,511,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,1023,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,1023,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,2047,0.015386667102575302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,2047,0.01651200031240781
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,4095,0.019610666980346043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,4095,0.01756799966096878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,8191,0.023391999304294586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,8191,0.023168000082174938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,16383,0.038133333126703896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,16383,0.031471999982992806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,32767,0.06025066475073496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,32767,0.05268799761931101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,65535,0.10025599598884583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,65535,0.08705600102742513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,1,0.06673599779605865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,float16,131071,0.18465065956115723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,1,0.06428266565004985
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,64,0,1,float16,fp8,131071,0.15902400016784668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,3,0.06471466521422069
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,3,0.06901333232720692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,7,0.06680533289909363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,7,0.06477333108584087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,15,0.06855999926726024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,15,0.0651093324025472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,31,0.06452799836794536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,63,0.06841599941253662
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,31,0.0681386689345042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,63,0.06473599870999654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,127,0.06966400146484375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,127,0.06611200173695882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,255,0.07098133365313213
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,255,0.06728533407052358
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,float16,511,0.10763733585675557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,64,0,1,float16,fp8,511,0.09745066364606221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,1,0.1263200044631958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,1,0.11983999609947205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,3,0.12571733196576437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,3,0.1199733316898346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,7,0.12594667077064514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,7,0.11957333485285442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,15,0.12602133552233377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,15,0.11981866757074992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,31,0.12626666824022928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,31,0.11980799833933513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,63,0.12626133362452188
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,63,0.11949867010116577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,127,0.12609066565831503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,127,0.12164800365765889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,1,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,1,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,3,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,3,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,7,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,7,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,float16,255,0.12890133261680603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,64,0,1,float16,fp8,255,0.12040533622105916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,15,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,15,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,31,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,31,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,63,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,127,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,127,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,511,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,511,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,1023,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,1023,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,2047,0.017573333034912746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,2047,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,4095,0.023200000325838726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,4095,0.021338666478792827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,8191,0.029578665892283123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,8191,0.03787733366092046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,16383,0.058287998040517174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,16383,0.05171200136343638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,32767,0.08633599678675334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,32767,0.10122666756312053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,1,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,3,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,3,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,7,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,fp8,65535,0.15474667151769003
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,7,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,64,0,1,float16,float16,65535,0.18188265959421793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,15,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,15,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,31,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,31,0.012693333129088083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,63,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,63,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,127,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,127,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,255,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,255,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,511,0.01471466695268949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,511,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,1023,0.017893332988023758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,1023,0.0170666662355264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,2047,0.023669332265853882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,2047,0.04032533367474874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,4095,0.039408000806967415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,4095,0.0599839985370636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,8191,0.10280533631642659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,8191,0.05992533266544342
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,16383,0.18509334325790405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,16383,0.10256000359853108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,1,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,1,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,float16,32767,0.350655992825826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,3,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,7,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,8,128,0,1,float16,fp8,32767,0.18179200092951456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,15,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,15,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,31,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,63,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,127,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,63,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,127,0.01071999967098236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,255,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,511,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,511,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,1023,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,2047,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,2047,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,4095,0.015749332805474598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,4095,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,8191,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,8191,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,16383,0.01807466646035512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,16383,0.0207893339296182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,32767,0.02346666653951009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,32767,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,65535,0.03794133414824804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,1,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,65535,0.02317333221435547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,1,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,3,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,3,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,7,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,7,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,15,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,15,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,31,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,63,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,fp8,131071,0.03982399900754293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,127,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,8,128,0,1,float16,float16,131071,0.059664001067479454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,127,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,255,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,255,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,511,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,511,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,1023,0.011754666765530905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,1023,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,2047,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,2047,0.014842666685581207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,4095,0.01699200024207433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,4095,0.016943999876578648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,8191,0.017407999684413273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,8191,0.01937599976857503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,16383,0.022917332748572033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,16383,0.02080533280968666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,32767,0.023706667125225067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,32767,0.03775466730197271
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,65535,0.03797333439191183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,65535,0.05852800110975901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,1,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,1,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,3,0.01469333345691363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,3,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,7,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,7,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,15,0.015381333728631338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,15,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,fp8,131071,0.05806933343410492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,8,128,0,1,float16,float16,131071,0.09769599636395772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,31,0.014789332946141561
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,31,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,63,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,63,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,127,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,127,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,255,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,511,0.018933333456516266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,255,0.01479999969402949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,511,0.017077332983414333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,1023,0.03409600009520849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,1023,0.021909333765506744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,2047,0.06052266558011373
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,2047,0.040175999204317726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,4095,0.10174399614334106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,4095,0.06078400214513143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,8191,0.18561599651972452
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,8191,0.10220799843470256
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,1,0.011855999628702799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,float16,16383,0.3516906499862671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,8,128,0,1,float16,fp8,16383,0.1837493379910787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,3,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,3,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,7,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,7,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,15,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,15,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,31,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,63,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,127,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,127,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,255,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,511,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,511,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,1023,0.012261333564917246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,1023,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,2047,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,2047,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,4095,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,4095,0.01670933390657107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,8191,0.021231998999913532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,8191,0.01913600042462349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,16383,0.03828266759713491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,16383,0.023423999547958374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,32767,0.059194669127464294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,32767,0.03796799977620443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,65535,0.10136000315348308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,1,0.019189332922299702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,1,0.019013332823912304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,65535,0.057349334160486855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,3,0.019578666736682255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,3,0.01777600000301997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,7,0.01937066639463107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,15,0.019248000035683315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,7,0.01764800027012825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,15,0.018895999838908512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,31,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,float16,131071,0.18459200859069824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,31,0.01913600042462349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,8,128,0,1,float16,fp8,131071,0.10016000270843506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,63,0.019306667149066925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,63,0.01747200017174085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,127,0.019093333433071773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,127,0.01907733331123988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,255,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,255,0.01894933357834816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,511,0.033333333830038704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,511,0.024005333582560223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,1023,0.052042668064435325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,1023,0.03510933369398117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,2047,0.09596266349156697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,2047,0.058101331194241844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,4095,0.17128000656763712
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,4095,0.0949173370997111
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,1,0.02743999908367793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,1,0.025407999753952026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,fp8,8191,0.16664000352223715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,8,128,0,1,float16,float16,8191,0.3253333369890849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,3,0.027658666173617046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,3,0.02532266577084859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,7,0.02752000093460083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,15,0.025487999121348064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,7,0.02533866713444392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,15,0.02759466568628947
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,31,0.027717334528764088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,31,0.025274666647116344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,63,0.025439999997615814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,63,0.027509334186712902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,127,0.027456000447273254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,127,0.02510400116443634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,255,0.033802665770053864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,255,0.02518400053183238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,511,0.0525439977645874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,511,0.03779733429352442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,1023,0.09110933542251587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,1023,0.05414933462937673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,float16,2047,0.17214399576187134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,8,128,0,1,float16,fp8,2047,0.09693866968154907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,1,0.04655999938646952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,3,0.045610666275024414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,1,0.040106666584809623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,3,0.0401706670721372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,7,0.046021332343419395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,7,0.039850667119026184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,15,0.0462719996770223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,15,0.039808000127474465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,31,0.045706664522488914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,63,0.04607999821503957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,31,0.04031466692686081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,63,0.03969600051641464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,127,0.047839999198913574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,127,0.04041599979003271
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,255,0.04208533465862274
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,255,0.05227733155091604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,511,0.09090666969617207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,511,0.060122668743133545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,1,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,float16,1023,0.1657493313153585
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,1,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,3,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,3,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,8,128,0,1,float16,fp8,1023,0.09351999560991923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,7,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,7,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,31,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,31,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,63,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,127,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,255,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,511,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,1023,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,1023,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,2047,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,2047,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,4095,0.021125334004561108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,4095,0.019823999454577763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,8191,0.038058665891488395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,8191,0.02385066697994868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,16383,0.05991999804973602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,16383,0.03996799886226654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,32767,0.10056533416112264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,32767,0.05861333509286245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,65535,0.18227199713389078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,65535,0.0972213347752889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,1,0.0812960018714269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,1,0.06850666801134746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,fp8,131071,0.17908267180124918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,8,128,0,1,float16,float16,131071,0.349727988243103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,3,0.06834666430950165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,3,0.08270399769147237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,7,0.08244266609350841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,7,0.06833600004514058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,15,0.08077333370844524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,15,0.0683840016523997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,31,0.08285866677761078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,31,0.06877866884072621
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,63,0.08255999783674876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,63,0.06855466465155284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,127,0.08205866813659668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,127,0.07208533088366191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,255,0.07238933444023132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,255,0.09106666843096416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,float16,511,0.16669867436091104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,8,128,0,1,float16,fp8,511,0.10733866691589355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,1,0.1316426694393158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,1,0.15217066804567972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,3,0.15255999565124512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,3,0.13212800025939941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,7,0.15196800231933594
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,7,0.13184533516565958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,15,0.15226667126019797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,15,0.13182933131853738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,31,0.1318719983100891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,31,0.1523413360118866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,63,0.15060800313949585
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,63,0.13225066661834717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,127,0.13196800152460733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,127,0.15029866496721903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,1,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,1,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,3,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,7,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,15,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,15,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,float16,255,0.16948266824086508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,8,128,0,1,float16,fp8,255,0.13409066200256348
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,31,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,63,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,63,0.01239466667175293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,127,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,127,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,511,0.012768000364303589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,511,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,1023,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,1023,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,2047,0.021205333371957142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,2047,0.01939733326435089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,4095,0.037776000797748566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,4095,0.02422933280467987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,8191,0.0377813329299291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,8191,0.06027733286221822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,16383,0.10116799672444661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,16383,0.057802667220433555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,32767,0.18307199080785116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,32767,0.09771200021107991
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,1,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,1,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,float16,65535,0.3482400178909302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,3,0.011717333147923151
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,7,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,3,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,8,128,0,1,float16,fp8,65535,0.17603200674057007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,15,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,15,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,31,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,127,0.014181333283583323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,255,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,255,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,511,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,511,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,1023,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,1023,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,2047,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,2047,0.019199999670187633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,4095,0.023552000522613525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,4095,0.021530665457248688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,8191,0.039690665900707245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,8191,0.031210665901501972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,16383,0.05995733539263407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,16383,0.05277866621812185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,32767,0.08716266353925069
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,32767,0.10097600022951762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,1,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,1,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,3,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,3,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,7,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,7,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,15,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,15,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,31,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,31,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,float16,65535,0.18467199802398682
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,127,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,255,0.010538666198650995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,64,0,1,float16,fp8,65535,0.15582933028539023
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,511,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,511,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,1023,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,1023,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,2047,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,2047,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,4095,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,4095,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,8191,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,8191,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,16383,0.01764800027012825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,16383,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,32767,0.021125334004561108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,32767,0.01940800001223882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,65535,0.021295999487241108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,65535,0.021301334102948506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,1,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,1,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,3,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,7,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,7,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,15,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,31,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,31,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,float16,131071,0.027658666173617046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,64,0,1,float16,fp8,131071,0.025621332228183746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,63,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,127,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,127,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,255,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,511,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,511,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,1023,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,2047,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,2047,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,4095,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,4095,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,8191,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,8191,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,16383,0.018277333428462345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,16383,0.01773333301146825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,32767,0.01929066702723503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,32767,0.018933333456516266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,65535,0.024133334557215374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,65535,0.0230880007147789
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,1,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,3,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,3,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,7,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,7,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,float16,131071,0.03999999910593033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,15,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,31,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,15,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,31,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,64,0,1,float16,fp8,131071,0.031471999982992806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,63,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,63,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,127,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,127,0.012741333494583765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,255,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,255,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,511,0.013365333278973898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,511,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,1023,0.01565333331624667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,1023,0.017375999440749485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,2047,0.023503998915354412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,2047,0.0234400009115537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,4095,0.04154666761557261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,4095,0.03148799886306127
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,8191,0.06147199869155884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,8191,0.05241066714127859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,16383,0.10398399829864502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,1,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,1,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,3,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,16383,0.08845333258310954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,7,0.011898666620254517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,15,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,float16,32767,0.18773865699768066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,31,0.010480000327030817
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,64,0,1,float16,fp8,32767,0.1564586659272512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,31,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,63,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,63,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,255,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,511,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,255,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,1023,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,1023,0.01268799975514412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,2047,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,2047,0.015344000111023584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,4095,0.01691199963291486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,4095,0.01720533271630605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,8191,0.017103999853134155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,8191,0.01801066721479098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,16383,0.02130666623512904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,16383,0.020853333175182343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,32767,0.02518400053183238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,32767,0.0233599990606308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,65535,0.03998400022586187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,65535,0.03179733455181122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,1,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,1,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,3,0.01481066644191742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,3,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,7,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,7,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,15,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,15,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,float16,131071,0.06005866825580597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,64,0,1,float16,fp8,131071,0.05385066568851471
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,31,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,31,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,63,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,63,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,127,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,127,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,255,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,255,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,511,0.017093333105246227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,511,0.01676799977819125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,1023,0.02165333429972331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,1023,0.02125866711139679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,2047,0.04046933352947235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,2047,0.03172266731659571
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,4095,0.0626986672480901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,4095,0.05421333511670431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,8191,0.10512533783912659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,8191,0.08973866701126099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,1,0.017488000293572743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,float16,16383,0.18866666158040366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,1,0.01788266624013583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,64,0,1,float16,fp8,16383,0.15679466724395752
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,3,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,3,0.017018667111794155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,7,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,7,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,15,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,15,0.017317333569129307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,31,0.017616000026464462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,31,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,63,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,63,0.017157333592573803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,127,0.017360000560681026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,127,0.01699200024207433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,255,0.017349333812793095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,255,0.017397332936525345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,511,0.023423999547958374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,511,0.02147199958562851
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,1023,0.035546667873859406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,1023,0.02977599948644638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,2047,0.058431997895240784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,2047,0.05234666665395101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,float16,4095,0.09713600079218547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,1,0.025706666211287182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,64,0,1,float16,fp8,4095,0.08153066535790761
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,3,0.024058667321999867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,1,0.025519999365011852
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,3,0.02508266766866048
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,7,0.02589333305756251
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,7,0.023605334262053173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,15,0.024874667326609295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,15,0.025759999950726826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,31,0.023578666150569916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,31,0.02366400013367335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,63,0.023658665517965954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,63,0.02534399926662445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,127,0.023306667804718018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,127,0.02366400013367335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,255,0.023402666052182514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,255,0.025407999753952026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,511,0.03696533292531967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,511,0.03364799916744232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,1023,0.054805333415667214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,1023,0.0499946673711141
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,3,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,float16,2047,0.09762666622797649
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,3,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,7,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,64,0,1,float16,fp8,2047,0.08504000306129456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,15,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,31,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,31,0.010687999427318573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,127,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,255,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,511,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,511,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,1023,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,2047,0.014720000326633453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,2047,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,4095,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,4095,0.016917333006858826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,8191,0.018778666853904724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,8191,0.01916266605257988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,16383,0.02312533309062322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,16383,0.023333333432674408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,32767,0.037871999045213066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,32767,0.03177600105603536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,65535,0.05227733155091604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,65535,0.0599839985370636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,1,0.037674665451049805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,1,0.03902933249870936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,fp8,131071,0.08966400225957234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,64,0,1,float16,float16,131071,0.1018346647421519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,3,0.039135999977588654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,3,0.03779733429352442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,7,0.038346665600935616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,7,0.03787733366092046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,15,0.03787733366092046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,15,0.037765334049860634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,31,0.039503999054431915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,31,0.03764266769091288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,63,0.03991466760635376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,63,0.038032000263532005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,127,0.038032000263532005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,127,0.03814399987459183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,255,0.04132800052563349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,255,0.03757333258787791
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,511,0.06078400214513143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,511,0.05436799923578898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,float16,1023,0.09590400258700053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,64,0,1,float16,fp8,1023,0.08547733227411906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,1,0.0664213349421819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,3,0.06803733110427856
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,1,0.06558933357397716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,3,0.0644053320089976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,7,0.06653333206971486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,7,0.06426666676998138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,15,0.0670826683441798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,31,0.06717866659164429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,31,0.06460799773534139
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,63,0.0669653316338857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,63,0.06482133269309998
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,127,0.0697813332080841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,127,0.06478933493296306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,255,0.07107733190059662
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,255,0.06637866795063019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,1,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,7,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,7,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,15,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,float16,511,0.10807999968528748
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,15,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,511,0.09795199831326802
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,31,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,63,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,63,0.01166933278242747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,127,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,127,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,255,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,511,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,511,0.013434667140245438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,1023,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,1023,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,2047,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,2047,0.016751999656359356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,4095,0.0195573332409064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,4095,0.018858666221300762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,8191,0.02348266790310542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,64,0,1,float16,fp8,15,0.06534400085608165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,8191,0.022602667411168415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,16383,0.03969600051641464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,16383,0.03211733450492223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,32767,0.05900266766548157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,32767,0.05225066840648651
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,65535,0.10145599643389384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,65535,0.0869653324286143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,3,0.011952000359694162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,7,0.011871999750534693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,15,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,15,0.012256000190973282
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,31,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,31,0.012042666474978128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,63,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,63,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,127,0.011701333026091257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,127,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,255,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,float16,131071,0.1838080088297526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,64,0,1,float16,fp8,131071,0.1586240033308665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,511,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,511,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,1023,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,1023,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,2047,0.020224000016848247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,2047,0.019354666272799175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,4095,0.03812800099452337
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,4095,0.023631999890009563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,8191,0.059861332178115845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,8191,0.03908266623814901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,16383,0.09934932986895244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,16383,0.05839466551939646
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,32767,0.09725866715113322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,32767,0.18170666694641113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,1,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,1,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,3,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,3,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,7,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,float16,65535,0.34399465719858807
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,7,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,15,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,31,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,4,128,0,1,float16,fp8,65535,0.17568532625834146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,127,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,255,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,127,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,255,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,511,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,511,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,1023,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,1023,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,2047,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,2047,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,4095,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,4095,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,8191,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,8191,0.016890666137139004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,16383,0.019333332777023315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,16383,0.019274666905403137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,32767,0.02179199953873952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,32767,0.02130666623512904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,65535,0.025381334125995636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,65535,0.022319999833901722
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,1,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,1,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,3,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,3,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,float16,131071,0.044405331214269005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,7,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,7,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,15,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,15,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,4,128,0,1,float16,fp8,131071,0.027621333797772724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,31,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,63,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,63,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,127,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,127,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,255,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,511,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,255,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,511,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,1023,0.011999999483426413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,1023,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,2047,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,2047,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,4095,0.015562667200962702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,4095,0.01716800034046173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,8191,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,8191,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,16383,0.01979200045267741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,16383,0.019205333044131596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,32767,0.024682665864626568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,32767,0.019381333142518997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,65535,0.03979733337958654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,65535,0.025487999121348064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,1,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,1,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,3,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,3,0.013631999492645264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,7,0.013610667238632837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,7,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,15,0.013487999637921652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,15,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,float16,131071,0.06115733087062836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,4,128,0,1,float16,fp8,131071,0.04190933207670847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,31,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,31,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,63,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,127,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,127,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,63,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,255,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,255,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,511,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,511,0.014906667172908783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,1023,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,1023,0.016735999534527462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,2047,0.03923200070858002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,2047,0.025258667767047882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,4095,0.060085331400235496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,4095,0.04008000095685323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,8191,0.06036800146102905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,8191,0.10215466221173604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,16383,0.10147733489672343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,16383,0.18321067094802856
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,1,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,1,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,3,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,fp8,32767,0.18203200896581015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,7,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,4,128,0,1,float16,float16,32767,0.34808532396952313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,7,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,15,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,31,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,63,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,127,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,127,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,255,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,255,0.011744000017642975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,511,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,511,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,1023,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,1023,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,2047,0.015552000453074774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,2047,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,4095,0.018197332819302876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,4095,0.017498667041460674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,8191,0.02272533377011617
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,8191,0.019018666197856266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,16383,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,16383,0.02370133250951767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,32767,0.04026666780312856
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,32767,0.025546667476495106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,65535,0.06117866436640421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,65535,0.04008000095685323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,1,0.015114666273196539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,1,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,float16,131071,0.10104533036549886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,3,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,7,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,3,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,7,0.014831999937693277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,15,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,4,128,0,1,float16,fp8,131071,0.06038400034109751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,15,0.014789332946141561
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,31,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,31,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,63,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,63,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,127,0.014671999961137772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,127,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,255,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,255,0.01482133318980535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,511,0.019088000059127808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,511,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,1023,0.03493333359559377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,2047,0.06044266621271769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,1023,0.022255999346574146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,2047,0.040192000567913055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,4095,0.10233066479365031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,4095,0.0617386649052302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,8191,0.10218133529027303
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,8191,0.18473599354426065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,1,0.01922133316596349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,1,0.017583999782800674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,3,0.01918399954835574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,3,0.018645333747069042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,fp8,16383,0.18541866540908813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,4,128,0,1,float16,float16,16383,0.35079999764760333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,7,0.019391999890406925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,7,0.019567999988794327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,15,0.019466667125622433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,15,0.018954666952292126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,31,0.019461333751678467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,31,0.018863999595244724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,63,0.019306667149066925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,63,0.01884799947341283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,127,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,127,0.018511999398469925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,255,0.018986667195955913
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,255,0.018768000106016796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,511,0.03365866591533025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,511,0.0234400009115537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,1023,0.053210665782292686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,1023,0.03554133325815201
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,2047,0.09629333019256592
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,2047,0.05904533465703329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,1,0.027653334041436512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,1,0.02535466601451238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,fp8,4095,0.09577066699663798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,4,128,0,1,float16,float16,4095,0.17091200749079385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,3,0.02754133443037669
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,3,0.025263999899228413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,7,0.027349332968393963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,7,0.025786665578683216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,15,0.02736533433198929
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,15,0.025466665625572205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,31,0.0276053324341774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,31,0.02569599946339925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,63,0.02741333345572154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,63,0.0252960001428922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,127,0.027130665878454845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,127,0.0252960001428922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,255,0.03182933231194814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,255,0.025493333737055462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,511,0.052282666166623436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,511,0.03688533355792364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,1023,0.08992000420888265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,1023,0.054773335655530296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,1,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,float16,2047,0.17139200369517008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,1,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,3,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,4,128,0,1,float16,fp8,2047,0.09660266836484273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,3,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,7,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,7,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,15,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,15,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,31,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,63,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,63,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,127,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,255,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,127,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,255,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,511,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,511,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,1023,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,1023,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,2047,0.015413332730531693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,2047,0.015552000453074774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,4095,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,4095,0.017594666530688603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,8191,0.021317332983016968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,8191,0.01932266727089882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,16383,0.039642666776975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,16383,0.023152001202106476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,32767,0.058490668733914696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,32767,0.039333333571751915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,65535,0.099589337905248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,65535,0.05853866537412008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,1,0.04475200176239014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,float16,131071,0.18508267402648926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,1,0.03988266736268997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,4,128,0,1,float16,fp8,131071,0.09974400202433269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,3,0.03974399964014689
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,3,0.045781334241231285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,7,0.046256000796953835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,7,0.04038933416207632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,15,0.04571199913819631
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,31,0.045994664231936135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,15,0.04064533362785975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,31,0.039887999494870506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,63,0.04562666515509287
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,63,0.04022400081157684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,127,0.048154667019844055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,127,0.03985599925120672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,255,0.052655999859174095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,255,0.042591998974482216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,511,0.0897706647713979
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,511,0.06083733340104421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,float16,1023,0.16497066617012024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,4,128,0,1,float16,fp8,1023,0.09504000345865886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,1,0.08189333478609721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,1,0.06985599795977275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,3,0.08188800017038982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,3,0.07061333457628886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,7,0.08167466521263123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,7,0.06921066840489705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,15,0.08100800216197968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,15,0.07084799806276958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,31,0.08309866487979889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,31,0.06980266670385997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,63,0.0812960018714269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,63,0.0707893321911494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,127,0.0830506682395935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,127,0.07256000240643819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,255,0.09173333644866943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,255,0.07297066847483318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,1,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,3,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,1,0.011493333925803503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,7,0.011834666132926941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,15,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,15,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,31,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,31,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,float16,511,0.1651040017604828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,4,128,0,1,float16,fp8,511,0.10611733794212341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,63,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,63,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,127,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,255,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,255,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,511,0.0120319997270902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,511,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,1023,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,1023,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,2047,0.017535999417304993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,2047,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,4095,0.021503999829292297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,4095,0.0194560003777345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,8191,0.039493332306543984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,8191,0.024069334069887798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,16383,0.0599839985370636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,16383,0.04010133445262909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,32767,0.1002506713072459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,32767,0.05952533086140951
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,65535,0.18308266003926596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,65535,0.09758933385213216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,1,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,1,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,3,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,3,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,7,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,7,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,float16,131071,0.34804801146189374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,15,0.011706666400035223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,15,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,31,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,31,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,63,0.011674666156371435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,63,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,127,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,127,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,4,128,0,1,float16,fp8,131071,0.17902400096257529
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,255,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,255,0.011802667131026586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,511,0.011792000383138657
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,511,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,1023,0.013605333864688873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,1023,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,2047,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,2047,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,4095,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,4095,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,8191,0.02332799881696701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,8191,0.025589334468046825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,16383,0.04042666653792063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,16383,0.031445334355036415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,32767,0.06078400214513143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,32767,0.054048001766204834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,65535,0.08689066767692566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,65535,0.10178132851918538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,1,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,1,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,3,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,7,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,3,0.01357866699496905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,7,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,15,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,15,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,31,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,31,0.011727999895811081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,63,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,63,0.01209066684047381
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,127,0.010522666076819101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,255,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,127,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,255,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,511,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,511,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,1023,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,1023,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,2047,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,4095,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,2047,0.011663999408483505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,4095,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,float16,131071,0.18594666322072348
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,64,0,1,float16,fp8,131071,0.15875732898712158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,8191,0.017338667064905167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,8191,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,16383,0.021568000316619873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,16383,0.021168000996112823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,32767,0.02369066576162974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,32767,0.023770667612552643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,65535,0.025274666647116344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,65535,0.025402667621771496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,1,0.010351999973257383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,1,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,3,0.010549332946538925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,7,0.010565333068370819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,3,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,15,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,31,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,31,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,127,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,63,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,float16,131071,0.02869333326816559
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,127,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,64,0,1,float16,fp8,131071,0.027109332382678986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,255,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,255,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,511,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,511,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,1023,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,1023,0.011749333391586939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,2047,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,2047,0.012618667135636011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,4095,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,4095,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,8191,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,8191,0.017583999782800674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,16383,0.01942933350801468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,16383,0.019717333217461903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,32767,0.021690666675567627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,32767,0.019365333020687103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,65535,0.02181866765022278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,65535,0.021664001047611237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,1,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,1,0.011957333733638128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,3,0.011493333925803503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,float16,131071,0.028463999430338543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,3,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,7,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,7,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,15,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,15,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,64,0,1,float16,fp8,131071,0.027637332677841187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,31,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,31,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,63,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,127,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,511,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,511,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,1023,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,1023,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,2047,0.017386666188637417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,2047,0.01806933308641116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,4095,0.023007998863856
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,4095,0.02145066608985265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,8191,0.03994666785001755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,8191,0.03123733401298523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,16383,0.05895466605822245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,16383,0.05233600238958994
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,32767,0.10178132851918538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,32767,0.08744532863299052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,3,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,7,0.010725333044926325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,3,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,15,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,31,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,63,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,63,0.012191999703645706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,127,0.010687999427318573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,127,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,255,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,511,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,255,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,float16,65535,0.1851466695467631
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,511,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,1023,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,1023,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,64,0,1,float16,fp8,65535,0.15667200088500977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,2047,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,2047,0.0136266661187013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,4095,0.01685333376129468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,4095,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,8191,0.017573333034912746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,8191,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,16383,0.01904533306757609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,16383,0.01923199991385142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,32767,0.021781332790851593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,32767,0.02161066730817159
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,65535,0.02532800038655599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,65535,0.023999998966852825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,1,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,1,0.012240000069141388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,3,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,3,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,7,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,7,0.012442667037248611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,15,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,15,0.012304000556468964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,fp8,131071,0.033701332906881966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,64,0,1,float16,float16,131071,0.04404800136884054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,31,0.011834666132926941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,31,0.012586666891972223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,63,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,63,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,127,0.011663999408483505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,255,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,255,0.01232533281048139
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,511,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,511,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,1023,0.016965333372354507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,1023,0.015775999675194424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,2047,0.025077333052953083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,2047,0.02170666555563609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,4095,0.041120000183582306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,4095,0.03194133440653483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,8191,0.061941335598627724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,8191,0.05342400074005127
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,16383,0.10450133681297302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,16383,0.0869653324286143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,1,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,1,0.014922666052977243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,3,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,float16,32767,0.18743467330932617
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,3,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,7,0.013653332988421122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,7,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,64,0,1,float16,fp8,32767,0.1567253371079763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,15,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,15,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,31,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,31,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,63,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,127,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,127,0.01479999969402949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,63,0.014138666292031607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,255,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,255,0.013349333157142004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,511,0.017114666601022083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,1023,0.023152001202106476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,511,0.017632000148296356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,1023,0.02141333371400833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,2047,0.04165333261092504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,2047,0.031983998914559685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,4095,0.0625600020090739
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,4095,0.05411200225353241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,1,0.017456000049908955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,1,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,float16,8191,0.10602666934331258
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,64,0,1,float16,fp8,8191,0.0883786678314209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,3,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,3,0.017360000560681026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,7,0.01749333366751671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,7,0.017184000462293625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,15,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,15,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,31,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,31,0.01718933383623759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,63,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,63,0.017407999684413273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,127,0.01749333366751671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,127,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,255,0.01752000053723653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,255,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,511,0.023050665855407715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,511,0.021381333470344543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,1023,0.03493333359559377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,1023,0.029546665648619335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,2047,0.058559998869895935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,2047,0.05055999755859375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,1,0.010698666175206503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,3,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,3,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,7,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,15,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,float16,4095,0.09774399797121684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,15,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,31,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,64,0,1,float16,fp8,4095,0.08124266564846039
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,63,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,31,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,63,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,127,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,127,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,255,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,255,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,511,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,1023,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,1023,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,2047,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,2047,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,4095,0.01692266638080279
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,4095,0.01727466657757759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,8191,0.018992000569899876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,8191,0.01749333366751671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,16383,0.02128000060717265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,16383,0.019834666202465694
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,32767,0.025429333249727886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,32767,0.023605334262053173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,65535,0.039877332746982574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,65535,0.03179733455181122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,1,0.02516266703605652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,1,0.023530667026837666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,float16,131071,0.06187733511130015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,64,0,1,float16,fp8,131071,0.05459733307361603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,3,0.025274666647116344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,3,0.023285334308942158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,7,0.025386666258176167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,7,0.024986666937669117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,15,0.023408000667889912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,15,0.023455999791622162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,31,0.025765334566434223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,31,0.025034666061401367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,63,0.023925334215164185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,63,0.02571733295917511
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,127,0.025546667476495106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,127,0.023599999646345775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,255,0.023919999599456787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,255,0.02568000058333079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,511,0.03674133370320002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,511,0.03221333275238673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,1023,0.05643733342488607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,1023,0.05017599960168203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,float16,2047,0.09862400094668071
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,1,0.03793599953254064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,64,0,1,float16,fp8,2047,0.08525866270065308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,3,0.038106667498747505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,3,0.03760000069936117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,1,0.039120001097520195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,7,0.04051200052102407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,7,0.03769599894682566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,15,0.03774933268626531
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,15,0.03814399987459183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,31,0.039936001102129616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,31,0.03808533400297165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,63,0.03958400090535482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,63,0.037952000896135964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,127,0.03977599988381068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,127,0.03796799977620443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,255,0.04200533529122671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,255,0.03830400109291077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,511,0.06047466893990835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,1,0.011936000237862269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,1,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,3,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,511,0.05580266813437144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,7,0.01071999967098236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,15,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,15,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,31,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,31,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,float16,1023,0.09553066889444987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,63,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,127,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,127,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,255,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,255,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,511,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,511,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,64,0,1,float16,fp8,1023,0.0851039985815684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,1023,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,1023,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,2047,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,2047,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,4095,0.015583999454975128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,4095,0.015418666104475657
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,8191,0.018901333212852478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,8191,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,16383,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,16383,0.025279998779296875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,32767,0.04018666595220566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,32767,0.031141333281993866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,65535,0.052095999320348106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,65535,0.05840000013510386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,1,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,3,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,3,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,7,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,7,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,15,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,31,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,31,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,63,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,63,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,127,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,127,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,255,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,255,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,fp8,131071,0.08956266442934673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,64,0,1,float16,float16,131071,0.1034879982471466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,511,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,511,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,1023,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,1023,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,2047,0.018933333456516266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,2047,0.017514667163292568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,4095,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,4095,0.021365332106749218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,8191,0.03982399900754293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,8191,0.025402667621771496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,16383,0.03977066775163015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,16383,0.060549333691596985
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,32767,0.1018293301264445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,32767,0.05894933144251505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,65535,0.18379199504852295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,65535,0.0993280013402303
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,1,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,7,0.01062400018175443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,7,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,15,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,31,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,63,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,127,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,255,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,255,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,float16,131071,0.34885867436726886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,511,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,511,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,1023,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,2047,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,1023,0.0116799995303154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,2047,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,2,128,0,1,float16,fp8,131071,0.18006932735443115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,4095,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,4095,0.012661332885424295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,8191,0.015413332730531693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,8191,0.016949333250522614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,16383,0.020207999895016353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,16383,0.021536000072956085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,32767,0.023562667270501454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,32767,0.021456000705560047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,65535,0.02603200078010559
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,65535,0.023733332753181458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,1,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,1,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,3,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,7,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,7,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,15,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,31,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,63,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,127,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,float16,131071,0.02979733298222224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,2,128,0,1,float16,fp8,131071,0.02978666623433431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,127,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,255,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,255,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,511,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,1023,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,1023,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,2047,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,2047,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,4095,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,4095,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,8191,0.01716800034046173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,8191,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,16383,0.019317333896954853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,16383,0.019466667125622433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,32767,0.021274665991465252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,32767,0.02102400114138921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,65535,0.02573866645495097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,65535,0.023413332800070446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,1,0.012335999558369318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,1,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,3,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,3,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,7,0.011733333269755045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,7,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,15,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,float16,131071,0.04540266593297323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,2,128,0,1,float16,fp8,131071,0.02897600084543228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,63,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,63,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,127,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,255,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,255,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,511,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,511,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,1023,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,1023,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,2047,0.02142400046189626
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,2047,0.018735999862353008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,4095,0.0394400010506312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,4095,0.023365333676338196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,8191,0.06054399907588959
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,8191,0.03939733405907949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,16383,0.10087466239929199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,16383,0.05860800047715505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,32767,0.1828320026397705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,32767,0.09875733653704326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,1,0.011637333780527115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,1,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,3,0.011733333269755045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,3,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,7,0.01394133393963178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,7,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,15,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,15,0.010698666175206503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,float16,65535,0.3442879915237427
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,31,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,2,128,0,1,float16,fp8,65535,0.17756267388661703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,31,0.011674666156371435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,63,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,63,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,127,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,127,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,255,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,255,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,511,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,511,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,1023,0.012693333129088083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,1023,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,2047,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,2047,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,4095,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,4095,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,8191,0.01942933350801468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,8191,0.018976000448067982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,16383,0.021359999974568684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,16383,0.019226666539907455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,32767,0.024400000770886738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,32767,0.021370666722456615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,65535,0.042250668009122215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,65535,0.025850666066010792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,1,0.012725333372751871
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,3,0.012778667112191519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,1,0.013957332819700241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,3,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,7,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,7,0.013855999956528345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,15,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,15,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,31,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,float16,131071,0.06277866661548615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,2,128,0,1,float16,fp8,131071,0.04408533374468485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,63,0.01259200026591619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,31,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,63,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,127,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,127,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,255,0.013429333766301474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,255,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,511,0.015279999623696009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,511,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,1023,0.017797333498795826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,1023,0.01594666639963786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,2047,0.039919999738534294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,2047,0.02609066665172577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,4095,0.0620959997177124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,4095,0.04170133173465729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,8191,0.10242666800816853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,8191,0.06222933530807495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,16383,0.1840160091718038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,16383,0.10122666756312053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,1,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,1,0.015520000209410986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,3,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,3,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,7,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,7,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,15,0.014826666563749313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,15,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,float16,32767,0.34966933727264404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,31,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,31,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,2,128,0,1,float16,fp8,32767,0.18290134270985922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,63,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,63,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,127,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,127,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,255,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,255,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,511,0.018725333114465077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,511,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,1023,0.03472000112136205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,1023,0.021157334248224895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,2047,0.06152533491452535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,2047,0.0418453315893809
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,4095,0.06205866734186808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,4095,0.10291199882825215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,1,0.019274666905403137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,1,0.017711999515692394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,float16,8191,0.18569066127141318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,3,0.019445333629846573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,3,0.017498667041460674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,2,128,0,1,float16,fp8,8191,0.10480533043543498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,7,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,7,0.019039999693632126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,15,0.019733333339293797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,31,0.019530666371186573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,15,0.019306667149066925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,31,0.019205333044131596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,63,0.019487999379634857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,63,0.019424000134070713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,127,0.019461333751678467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,127,0.018826667219400406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,255,0.019482667247454327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,255,0.01791999985774358
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,511,0.03364799916744232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,511,0.023573334018389385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,1023,0.03381866713364919
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,1023,0.053397332628568016
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,2047,0.09804266691207886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,2047,0.059258664647738137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,1,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,1,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,3,0.010533332824707031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,3,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,float16,4095,0.17378666003545126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,7,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,15,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,15,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,31,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,31,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,2,128,0,1,float16,fp8,4095,0.09764800469080608
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,63,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,63,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,127,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,127,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,255,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,255,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,511,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,511,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,1023,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,1023,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,2047,0.016864000509182613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,2047,0.01563199982047081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,4095,0.017125333348910015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,4095,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,8191,0.01947733387351036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,8191,0.019189332922299702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,16383,0.023520000278949738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,16383,0.02111999938885371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,32767,0.04181333382924398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,32767,0.025818665822347004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,65535,0.060517330964406334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,65535,0.04030400017897288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,1,0.027424000203609467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,1,0.025957333544890087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,float16,131071,0.10143466790517171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,2,128,0,1,float16,fp8,131071,0.06263466676076253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,3,0.027935999135176342
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,3,0.02550400048494339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,7,0.027552001178264618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,7,0.025621332228183746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,15,0.027503999571005504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,15,0.02584533393383026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,31,0.02754666656255722
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,31,0.025279998779296875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,63,0.02754133443037669
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,63,0.025263999899228413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,127,0.027786667148272198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,127,0.025333332518736523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,255,0.03356799980004629
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,255,0.025557334224383037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,511,0.05222400029500326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,511,0.03770133356253306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,1023,0.09205866853396098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,1023,0.05457599957784017
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,float16,2047,0.17486933867136636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,2,128,0,1,float16,fp8,2047,0.09853333234786987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,1,0.0458186666170756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,1,0.04008000095685323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,3,0.04597333570321401
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,3,0.04040000090996424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,7,0.04584000011285146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,7,0.039493332306543984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,15,0.04598399996757507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,15,0.04155733436346054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,31,0.04619200030962626
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,31,0.03996799886226654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,63,0.045696000258127846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,63,0.04005333284536997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,127,0.04774933556715647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,127,0.03967999915281931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,255,0.052485331892967224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,255,0.043920000394185386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,511,0.09095999598503113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,511,0.06044266621271769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,1,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,1,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,3,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,3,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,7,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,7,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,15,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,15,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,31,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,float16,1023,0.1673120061556498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,63,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,127,0.010687999427318573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,2,128,0,1,float16,fp8,1023,0.09363200267155965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,255,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,511,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,511,0.011727999895811081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,1023,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,1023,0.011722666521867117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,2047,0.01781333362062772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,2047,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,4095,0.017605333278576534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,4095,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,8191,0.02146133283774058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,8191,0.019434666881958645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,16383,0.023647998770078022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,16383,0.04038399954636892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,32767,0.060266668597857155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,32767,0.039733332892258964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,65535,0.10148266951243083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,65535,0.058575997749964394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,1,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,1,0.01184533288081487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,3,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,7,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,float16,131071,0.1867306629816691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,7,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,15,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,15,0.01181866725285848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,31,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,31,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,63,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,63,0.01258133351802826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,127,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,127,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,255,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,2,128,0,1,float16,fp8,131071,0.09962133566538493
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,255,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,511,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,511,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,1023,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,1023,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,2047,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,2047,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,4095,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,4095,0.016837333639462788
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,8191,0.01945066700379054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,8191,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,16383,0.0233599990606308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,16383,0.023205332458019257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,32767,0.03994666785001755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,32767,0.031557333966096245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,65535,0.0603413333495458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,65535,0.05209066470464071
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,1,0.010677333921194077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,1,0.010522666076819101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,3,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,3,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,7,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,7,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,15,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,31,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,15,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,31,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,63,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,127,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,255,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,255,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,511,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,1023,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,511,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,1023,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,2047,0.01163200040658315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,2047,0.01232533281048139
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,4095,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,4095,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,float16,131071,0.10407466689745586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,8191,0.01532799998919169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,64,0,1,float16,fp8,131071,0.08958400289217631
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,8191,0.017173333714405697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,16383,0.019461333751678467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,16383,0.02144533395767212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,32767,0.027701333165168762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,32767,0.027429332335789997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,65535,0.033215999603271484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,1,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,65535,0.03197866678237915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,1,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,3,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,7,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,7,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,15,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,float16,131071,0.03467733412981033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,64,0,1,float16,fp8,131071,0.0353973334034284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,15,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,31,0.010522666076819101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,31,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,63,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,63,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,127,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,127,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,255,0.010165333126982054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,511,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,511,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,1023,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,1023,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,2047,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,2047,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,4095,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,4095,0.01350933313369751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,8191,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,8191,0.017018667111794155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,16383,0.021354667842388153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,16383,0.01934933289885521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,32767,0.023455999791622162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,32767,0.023178666830062866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,65535,0.023605334262053173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,65535,0.025392000873883564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,1,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,3,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,float16,131071,0.028058665494124096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,7,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,15,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,31,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,64,0,1,float16,fp8,131071,0.02777066578467687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,63,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,127,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,255,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,255,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,127,0.01228800043463707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,511,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,511,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,1023,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,1023,0.014906667172908783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,2047,0.016965333372354507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,2047,0.016778666526079178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,4095,0.01934933289885521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,4095,0.01950399950146675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,8191,0.02333866556485494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,8191,0.023413332800070446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,16383,0.03979199876387914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,16383,0.03160533308982849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,32767,0.05994133154551188
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,32767,0.05232533315817515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,65535,0.08692266543706258
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,65535,0.10147200028101604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,1,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,3,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,7,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,15,0.011610666910807291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,31,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,31,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,127,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,63,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,127,0.010565333068370819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,255,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,255,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,511,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,1023,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,1023,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,2047,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,2047,0.013712000101804733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,4095,0.014815999815861383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,8191,0.01939733326435089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,4095,0.015530666957298914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,8191,0.01874133323629697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,16383,0.019754666835069656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,16383,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,fp8,131071,0.15914666652679443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,64,0,1,float16,float16,131071,0.1857973337173462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,32767,0.02143999934196472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,32767,0.021002667645613354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,65535,0.023237332701683044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,65535,0.023050665855407715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,1,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,1,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,3,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,7,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,float16,131071,0.02976000060637792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,64,0,1,float16,fp8,131071,0.02754133443037669
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,7,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,15,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,15,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,31,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,63,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,63,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,127,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,127,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,255,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,255,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,511,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,511,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,1023,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,1023,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,2047,0.01940800001223882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,2047,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,4095,0.023434666295846302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,4095,0.023317334552605946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,8191,0.03946666667858759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,8191,0.03173866619666418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,16383,0.06002133091290792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,16383,0.05230399966239929
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,32767,0.10193600257237752
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,1,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,32767,0.08745599786440532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,1,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,3,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,3,0.011941333611806234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,7,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,7,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,15,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,15,0.012703999876976013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,31,0.013487999637921652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,31,0.011994666109482447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,63,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,63,0.011920000116030375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,float16,65535,0.18381865819295248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,127,0.013434667140245438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,127,0.012469333906968435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,255,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,255,0.011594666788975397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,511,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,511,0.01340266689658165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,1023,0.01563199982047081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,1023,0.01568000018596649
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,2047,0.025290665527184803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,2047,0.023237332701683044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,64,0,1,float16,fp8,65535,0.15679466724395752
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,4095,0.042591998974482216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,4095,0.03196800003449122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,8191,0.06226666768391927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,1,0.014831999937693277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,8191,0.05284800132115682
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,1,0.014906667172908783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,3,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,3,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,7,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,float16,16383,0.105295995871226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,64,0,1,float16,fp8,16383,0.08742400010426839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,7,0.01360000049074491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,15,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,15,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,31,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,31,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,63,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,63,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,127,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,127,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,255,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,255,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,511,0.017237332959969837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,511,0.01687466725707054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,1023,0.021194666624069214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,1023,0.021221332252025604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,2047,0.04038399954636892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,2047,0.03349866718053818
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,4095,0.062362665931383766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,1,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,4095,0.05481599768002828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,3,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,7,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,7,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,15,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,15,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,31,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,31,0.011893333246310553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,63,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,255,0.010399999717871347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,127,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,float16,8191,0.10569600264231364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,64,0,1,float16,fp8,8191,0.08965333302815755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,511,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,511,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,1023,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,1023,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,2047,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,2047,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,4095,0.017050666113694508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,4095,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,8191,0.017173333714405697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,8191,0.01754133279124896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,16383,0.0194560003777345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,16383,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,32767,0.021365332106749218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,32767,0.020975999534130096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,65535,0.025413334369659424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,65535,0.025413334369659424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,1,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,float16,131071,0.04359466830889384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,1,0.017125333348910015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,64,0,1,float16,fp8,131071,0.03342933456103007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,3,0.017488000293572743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,3,0.016997333616018295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,7,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,7,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,15,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,31,0.01727466657757759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,15,0.017386666188637417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,31,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,63,0.017045332739750545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,63,0.0170666662355264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,127,0.017114666601022083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,127,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,255,0.01709866647919019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,255,0.017290666699409485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,511,0.021759999295075733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,511,0.021520001192887623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,1023,0.035386666655540466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,1023,0.02974933385848999
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,2047,0.05820266902446747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,2047,0.05068266888459524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,1,0.025493333737055462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,1,0.02380266785621643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,float16,4095,0.09788266817728679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,64,0,1,float16,fp8,4095,0.08170133332411449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,3,0.025216000775496166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,3,0.023738667368888855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,7,0.02571733295917511
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,7,0.02351466566324234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,15,0.025957333544890087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,15,0.02382933348417282
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,31,0.023925334215164185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,31,0.02403733382622401
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,63,0.02515733242034912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,63,0.023685333629449207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,127,0.02386133372783661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,127,0.023503998915354412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,255,0.025290665527184803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,255,0.024826665719350178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,511,0.037445334096749626
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,511,0.03225066761175791
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,1023,0.05611733098824819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,1,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,1023,0.05089599887530009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,3,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,7,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,7,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,15,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,15,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,31,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,31,0.011493333925803503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,63,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,127,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,63,0.011578666667143503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,127,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,255,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,255,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,511,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,511,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,1023,0.012410666793584824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,1023,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,float16,2047,0.09804800152778625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,2047,0.015493333339691162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,2047,0.015583999454975128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,4095,0.018730666488409042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,64,0,1,float16,fp8,2047,0.08550399541854858
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,4095,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,8191,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,8191,0.017770666629076004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,16383,0.021242665747801464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,16383,0.019359999646743137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,32767,0.025285333395004272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,32767,0.023760000864664715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,65535,0.0314026673634847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,65535,0.041759997606277466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,1,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,1,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,3,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,3,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,7,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,7,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,31,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,fp8,131071,0.05399466554323832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,64,0,1,float16,float16,131071,0.06135466694831848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,63,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,127,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,127,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,255,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,511,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,511,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,1023,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,1023,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,2047,0.015610666324694952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,2047,0.015637333194414776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,4095,0.019445333629846573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,4095,0.016906666258970898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,8191,0.02187199890613556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,8191,0.01950399950146675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,16383,0.04067733387152354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,16383,0.024842667082945507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,32767,0.061936000982920326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,32767,0.03967999915281931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,65535,0.10169600447018941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,65535,0.06069866816202799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,1,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,1,0.011770666887362799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,3,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,3,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,7,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,7,0.012304000556468964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,15,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,31,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,15,0.012245333443085352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,63,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,63,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,127,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,127,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,float16,131071,0.18717332681020102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,255,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,511,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,511,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,1023,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,2047,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,1023,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,2047,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,4095,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,4095,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,8191,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,8191,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,16383,0.021104000508785248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,16383,0.020096000283956528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,1,1,128,0,1,float16,fp8,131071,0.10099732875823975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,32767,0.028602667152881622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,32767,0.027482666075229645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,65535,0.034272000193595886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,65535,0.030085332691669464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,1,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,1,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,3,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,3,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,7,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,7,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,15,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,15,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,31,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,31,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,float16,131071,0.037978666524092354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,63,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,1,1,128,0,1,float16,fp8,131071,0.03552533437808355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,127,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,255,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,511,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,511,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,1023,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,1023,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,2047,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,2047,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,4095,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,4095,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,8191,0.01710933322707812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,8191,0.01704000060757001
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,16383,0.021087999145189922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,16383,0.022410665949185688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,32767,0.025072000920772552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,32767,0.025029333929220837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,65535,0.025360000630219776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,65535,0.027285332481066387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,1,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,1,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,3,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,float16,131071,0.03121600051720937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,7,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,15,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,7,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,31,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,1,1,128,0,1,float16,fp8,131071,0.029482667644818623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,63,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,127,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,127,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,255,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,255,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,511,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,511,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,1023,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,1023,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,2047,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,2047,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,4095,0.021541332205136616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,4095,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,8191,0.03984533250331879
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,8191,0.025450666745503742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,16383,0.061349332332611084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,16383,0.03961600114901861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,32767,0.10229333241780598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,32767,0.058037335673967995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,65535,0.1832159956296285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,65535,0.09950400392214458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,1,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,1,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,3,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,7,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,3,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,15,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,31,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,31,0.01179733375708262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,63,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,63,0.01173866664369901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,127,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,255,0.010474666953086853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,127,0.011695999652147293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,255,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,511,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,511,0.011829332758982977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,1023,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,1023,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,2047,0.014767999450365702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,4095,0.015306666493415833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,2047,0.013642666240533194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,4095,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,8191,0.019632000476121902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,8191,0.01749333366751671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,float16,131071,0.34834134578704834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,16383,0.021066665649414062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,16383,0.02163200080394745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,1,1,128,0,1,float16,fp8,131071,0.18197866280873617
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,32767,0.023413332800070446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,32767,0.02378133436044057
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,65535,0.026026666164398193
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,65535,0.02329600105683009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,1,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,3,0.011727999895811081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,7,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,7,0.011690666278203329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,float16,131071,0.04814399778842926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,1,1,128,0,1,float16,fp8,131071,0.0332640012105306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,31,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,31,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,63,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,63,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,127,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,255,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,255,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,511,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,511,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,1023,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,1023,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,2047,0.02125866711139679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,2047,0.018826667219400406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,4095,0.023333333432674408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,4095,0.03997333347797394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,8191,0.03953066716591517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,8191,0.06053866446018219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,16383,0.060165335734685264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,16383,0.10203733046849568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,32767,0.1818293333053589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,32767,0.09860799709955852
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,1,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,1,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,3,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,3,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,7,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,7,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,15,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,float16,65535,0.3444639841715495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,15,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,31,0.013408000270525614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,63,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,31,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,1,1,128,0,1,float16,fp8,65535,0.17733333508173624
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,63,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,127,0.01351999988158544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,127,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,255,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,511,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,511,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,255,0.013845333208640417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,1023,0.019226666539907455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,1023,0.017317333569129307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,2047,0.04062933226426443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,2047,0.024293333292007446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,4095,0.062352001667022705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,4095,0.041989331444104515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,8191,0.10340266426404317
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,8191,0.06115200122197469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,1,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,3,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,1,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,3,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,float16,16383,0.18390933672587076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,7,0.015370666980743408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,1,1,128,0,1,float16,fp8,16383,0.10387200117111206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,7,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,15,0.01553600033124288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,15,0.014730667074521383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,31,0.014954666296641031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,63,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,31,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,63,0.014752000570297241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,127,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,127,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,255,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,255,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,511,0.01915733392039935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,511,0.017514667163292568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,1023,0.035317334036032356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,1023,0.02276266614596049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,2047,0.06121066709359487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,2047,0.04170133173465729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,4095,0.10367467006047566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,4095,0.06311466793219249
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,1,0.012719999998807907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,1,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,3,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,3,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,7,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,7,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,15,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,15,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,31,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,63,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,127,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,float16,8191,0.18636266390482584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,127,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,255,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,255,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,511,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,511,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,1023,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,1023,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,1,1,128,0,1,float16,fp8,8191,0.10551466544469197
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,2047,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,2047,0.015605332950750986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,4095,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,4095,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,8191,0.019274666905403137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,8191,0.01959466685851415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,16383,0.023365333676338196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,16383,0.02054399996995926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,32767,0.025306666890780132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,32767,0.021829334398110706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,65535,0.04299733539422353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,65535,0.02550400048494339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,1,0.01932799940307935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,1,0.018992000569899876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,float16,131071,0.06443733473618825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,1,1,128,0,1,float16,fp8,131071,0.04389866689840952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,3,0.018986667195955913
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,3,0.019039999693632126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,7,0.019386666516462963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,7,0.01758933315674464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,15,0.01960533360640208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,15,0.0191040001809597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,31,0.019333332777023315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,31,0.01877333347996076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,63,0.019978666057189304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,63,0.018757333358128864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,127,0.019658666104078293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,127,0.019632000476121902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,255,0.01894933357834816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,255,0.017594666530688603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,511,0.03363733241955439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,511,0.023210667073726654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,1023,0.052389333645502724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,1023,0.03531199942032496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,2047,0.0974720021088918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,2047,0.05862933397293091
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,float16,4095,0.17542399962743124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,1,0.02805333336194356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,1,1,128,0,1,float16,fp8,4095,0.0962720016638438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,1,0.026288000245889027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,3,0.025573333104451496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,3,0.027285332481066387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,7,0.02771199991305669
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,7,0.025829332570234936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,15,0.02513066679239273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,15,0.027322667340437572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,31,0.027285332481066387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,31,0.025472000241279602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,63,0.02717333287000656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,63,0.02526933451493581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,127,0.027248000105222065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,127,0.02535466601451238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,255,0.03165333221356074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,255,0.025429333249727886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,511,0.052373334765434265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,511,0.037578667203585304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,1023,0.09124799569447835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,1023,0.05479466418425242
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,1,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,1,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,3,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,7,0.011600000162919363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,15,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,31,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,31,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,63,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,63,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,127,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,127,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,255,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,511,0.011765333513418833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,511,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,1023,0.011978667229413986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,float16,2047,0.17451733350753784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,1023,0.013621332744757334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,2047,0.015568000574906668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,2047,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,4095,0.01770666614174843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,4095,0.017605333278576534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,8191,0.020842666427294414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,8191,0.019765333582957584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,16383,0.02489600082238515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,1,1,128,0,1,float16,fp8,2047,0.0978613297144572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,16383,0.021231998999913532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,32767,0.041984001795450844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,32767,0.025237334271272022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,65535,0.06020799775918325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,65535,0.04178666571776072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,1,0.04818133513132731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,1,0.04804799954096476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,1,0.0406986673672994
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,1,0.040031999349594116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,3,0.04781866570313772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,3,0.04825599988301595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,3,0.03975466638803482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,float16,131071,0.10155733426411946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,3,0.03995733211437861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,1,1,128,0,1,float16,fp8,131071,0.06163200239340464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,7,0.048325334986050926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,7,0.048112000028292336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,7,0.04020266731580099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,15,0.05009600023428599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,15,0.04196266829967499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,15,0.042693331837654114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,31,0.05879466732343038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,31,0.06047466893990835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,31,0.05229333539803823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,7,0.04141866664091746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,31,0.05203199883302053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,15,0.05020800232887268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,63,0.06017066538333893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,63,0.05981333553791046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,63,0.05246399839719137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,63,0.05206400156021118
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,127,0.05990933378537496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,127,0.060133333007494606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,127,0.053818667928377785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,255,0.06001066664854685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,127,0.05211733281612396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,255,0.05376000205675761
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,255,0.0699946681658427
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,255,0.06271466612815857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,511,0.06033066908518473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,511,0.10291199882825215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,511,0.05227733155091604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,511,0.09687999884287517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,1023,0.060458665092786155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,1023,0.05249066650867462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,1023,0.1671253244082133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,1023,0.1618826687335968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,2047,0.06065600117047628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,2047,0.05411200225353241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,2047,0.30049065748850506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,2047,0.2940693298975627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,float16,4095,0.06066133578618368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,128,1,float16,fp8,4095,0.052255998055140175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,float16,4095,0.5668373505274454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,1,0.04789866507053375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,1,0.04819199939568838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,64,0,1,float16,fp8,4095,0.5581760009129842
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,1,0.04025600105524063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,1,0.03982399900754293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,3,0.04864533245563507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,3,0.04775999983151754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,3,0.04057066639264425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,7,0.04957866668701172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,3,0.04022933294375738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,7,0.04854933420817057
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,7,0.04204266766707102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,7,0.04164266586303711
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,15,0.05036266644795736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,15,0.049839998284975685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,15,0.04299733539422353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,15,0.04229333500067393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,31,0.06025066475073496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,31,0.058320000767707825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,31,0.05235200126965841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,31,0.0521066685517629
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,63,0.060362666845321655
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,63,0.05986666679382324
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,63,0.0518506666024526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,63,0.05240533252557119
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,127,0.06020799775918325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,127,0.060218666990598045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,127,0.05384533107280731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,127,0.05262400209903717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,255,0.06031466523806254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,255,0.05203199883302053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,255,0.0703306645154953
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,255,0.06384000182151794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,511,0.060133333007494606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,511,0.1032533347606659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,511,0.05394133428732554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,511,0.09701866904894511
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,1023,0.06019733349482218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,1023,0.16726400454839072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,1023,0.05320000151793162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,1023,0.1613866686820984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,2047,0.06019733349482218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,2047,0.054005334774653115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,2047,0.30061866839726764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,2047,0.2932159900665283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,float16,4095,0.06070399781068166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,128,1,float16,fp8,4095,0.052522664268811546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,1,0.04807466765244802
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,float16,4095,0.5669546524683634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,1,0.040074666341145836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,1,0.04837866624196371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,64,0,1,float16,fp8,4095,0.5577760140101115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,1,0.03975466638803482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,3,0.04818666477998098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,3,0.04827733337879181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,3,0.03982933362325033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,3,0.03991466760635376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,7,0.04817600051561991
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,7,0.04165866722663244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,7,0.04987733562787374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,15,0.04979733129342397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,7,0.0421066681543986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,15,0.04194133480389913
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,15,0.05022400120894114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,15,0.04222933451334635
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,31,0.058837334314982094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,31,0.05866133173306783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,31,0.05209066470464071
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,31,0.052154665191968284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,63,0.05964266757170359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,63,0.05227200190226237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,63,0.0598880002895991
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,63,0.05225066840648651
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,127,0.06018666426340739
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,127,0.060271998246510826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,127,0.05193066596984863
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,127,0.05197333296140035
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,255,0.052154665191968284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,255,0.07042666773001353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,255,0.06044800082842509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,255,0.06257600088914235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,511,0.060559997955958046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,511,0.10315733154614766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,511,0.052389333645502724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,511,0.09692266583442688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,1023,0.06043200194835663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,1023,0.1672053337097168
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,1023,0.05287466446558634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,1023,0.16160000363985697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,2047,0.06070399781068166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,2047,0.05423999826113383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,2047,0.3002133369445801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,2047,0.29412267605463666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,float16,4095,0.06042666733264923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,128,1,float16,fp8,4095,0.051967998345692955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,float16,4095,0.5747520128885905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,1,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,1,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,64,0,1,float16,fp8,4095,0.5574719905853271
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,1,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,1,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,3,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,3,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,3,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,3,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,7,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,7,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,7,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,7,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,15,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,15,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,15,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,15,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,31,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,31,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,31,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,31,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,63,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,63,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,63,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,63,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,127,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,127,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,127,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,127,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,255,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,255,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,255,0.014597332725922266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,255,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,511,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,511,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,511,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,511,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,1023,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,1023,0.01699200024207433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,1023,0.014373333503802618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,1023,0.017173333714405697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,2047,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,2047,0.027376001079877216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,2047,0.01609066625436147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,2047,0.025194667279720306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,float16,4095,0.015487999965747198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,float16,4095,0.04274133344491323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,128,1,float16,fp8,4095,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,1,0.009066666786869368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,64,0,1,float16,fp8,4095,0.03585066646337509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,1,0.00922133338948091
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,1,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,3,0.00922133338948091
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,3,0.008810666700204214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,3,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,7,0.008757333581646284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,7,0.009061333412925402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,7,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,15,0.009279999881982803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,15,0.009306666751702627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,15,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,15,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,31,0.010581333190202713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,31,0.008858666444818178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,31,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,63,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,63,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,127,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,63,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,127,0.010464000205198923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,127,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,127,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,255,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,255,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,255,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,255,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,511,0.01969066634774208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,511,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,511,0.013487999637921652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,511,0.02250133454799652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,1023,0.02077866718173027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,1023,0.023423999547958374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,1023,0.02515200028816859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,1023,0.027717334528764088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,2047,0.020224000016848247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,2047,0.029493334392706554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,2047,0.02516799916823705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,2047,0.031498665610949196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,4095,0.019461333751678467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,4095,0.042021334171295166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,4095,0.024714666108290356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,4095,0.0418453315893809
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,8191,0.02103466788927714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,8191,0.02348266790310542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,8191,0.064560001095136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,8191,0.062218666076660156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,float16,16383,0.019343999524911244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,float16,16383,0.11157866319020589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,128,1,float16,fp8,16383,0.025146665672461193
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,1,0.009226666763424873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,1,0.00919999989370505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,1,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,64,0,1,float16,fp8,16383,0.09916800260543823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,1,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,3,0.009290666629870733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,3,0.00878399983048439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,3,0.010602666685978571
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,3,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,7,0.008821333448092142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,7,0.009183999771873156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,7,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,7,0.010591999938090643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,15,0.009125333279371262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,15,0.009098666409651438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,15,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,15,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,31,0.009103999783595404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,31,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,31,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,63,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,63,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,127,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,127,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,127,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,127,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,255,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,255,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,255,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,255,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,511,0.021168000996112823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,511,0.02117866774400075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,511,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,511,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,1023,0.019205333044131596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,1023,0.025226667523384094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,1023,0.02502399931351344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,1023,0.02777066578467687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,2047,0.019733333339293797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,2047,0.029930666089057922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,2047,0.023541333774725597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,2047,0.03183466692765554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,4095,0.020666666328907013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,4095,0.04186666508515676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,4095,0.023221333821614582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,4095,0.04234666625658671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,8191,0.01979200045267741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,8191,0.06412266691525777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,8191,0.025226667523384094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,8191,0.0621919979651769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,float16,16383,0.02125866711139679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,float16,16383,0.11130666732788086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,128,1,float16,fp8,16383,0.023397333920001984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,1,0.009365333244204521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,1,0.008810666700204214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,64,0,1,float16,fp8,16383,0.101200004418691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,1,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,1,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,3,0.009194666519761086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,3,0.008842666943868002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,3,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,3,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,7,0.009232000137368837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,7,0.008938666433095932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,7,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,15,0.008890666688481966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,15,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,15,0.010496000448862711
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,31,0.008832000195980072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,31,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,63,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,63,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,31,0.011893333246310553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,63,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,63,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,127,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,127,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,127,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,127,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,255,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,255,0.011663999408483505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,255,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,255,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,511,0.021877333521842957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,511,0.021546666820844013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,511,0.013429333766301474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,511,0.013946666071812311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,1023,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,1023,0.024330665667851765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,1023,0.029557332396507263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,1023,0.02568000058333079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,2047,0.019823999454577763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,2047,0.031173333525657654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,2047,0.0315786674618721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,2047,0.025861332813898723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,4095,0.021568000316619873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,4095,0.024170666933059692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,4095,0.0429066667954127
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,4095,0.042133331298828125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,8191,0.06609066824118297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,8191,0.019658666104078293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,8191,0.024661332368850708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,8191,0.06247466802597046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,float16,16383,0.019274666905403137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,float16,16383,0.11166399717330933
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,128,1,float16,fp8,16383,0.025146665672461193
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,1,0.011727999895811081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,64,0,1,float16,fp8,16383,0.10195199648539226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,1,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,1,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,1,0.011898666620254517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,3,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,3,0.010549332946538925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,3,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,7,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,3,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,7,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,7,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,15,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,15,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,15,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,31,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,31,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,31,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,63,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,63,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,63,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,63,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,127,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,255,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,127,0.012074666718641916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,255,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,255,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,255,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,511,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,511,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,511,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,511,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,1023,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,1023,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,1023,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,1023,0.01257066677014033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,2047,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,2047,0.011722666521867117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,2047,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,2047,0.012309333930412928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,4095,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,4095,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,4095,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,4095,0.015893333901961643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,8191,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,8191,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,8191,0.01732800031701724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,8191,0.017984000345071156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,float16,16383,0.01747200017174085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,float16,16383,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,128,1,float16,fp8,16383,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,1,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,1,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,1,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,64,0,1,float16,fp8,16383,0.018863999595244724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,1,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,3,0.014752000570297241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,3,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,3,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,7,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,7,0.013637332866589228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,3,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,7,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,7,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,15,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,15,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,15,0.010687999427318573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,31,0.014773332824309668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,31,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,31,0.01071999967098236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,63,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,63,0.01695999999841054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,63,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,63,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,127,0.016842667013406754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,127,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,127,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,127,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,255,0.01553600033124288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,255,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,255,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,255,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,511,0.016250666230916977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,511,0.017994667092959087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,511,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,511,0.015594666202863058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,1023,0.016879999389251072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,1023,0.02327466756105423
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,1023,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,1023,0.019567999988794327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,2047,0.01588800052801768
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,2047,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,2047,0.03379199902216593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,2047,0.030085332691669464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,4095,0.016789333273967106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,4095,0.054671997825304665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,4095,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,4095,0.05023466547330221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,8191,0.016805333395799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,8191,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,8191,0.09629333019256592
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,8191,0.09108266234397888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,float16,16383,0.016821333517630894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,float16,16383,0.17945067087809244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,128,1,float16,fp8,16383,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,64,0,1,float16,fp8,16383,0.17084266742070517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,1,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,1,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,1,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,1,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,3,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,3,0.013877333452304205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,3,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,3,0.009248000259200731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,7,0.014544000228246054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,7,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,7,0.010389333590865135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,7,0.010512000570694605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,15,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,15,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,15,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,15,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,31,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,31,0.01479999969402949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,31,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,31,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,63,0.01544533297419548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,63,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,63,0.016970666746298473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,63,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,127,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,127,0.016778666526079178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,127,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,127,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,255,0.01672533278663953
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,255,0.01584533353646596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,255,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,255,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,511,0.015743999431530636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,511,0.017338667064905167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,511,0.014842666685581207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,511,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,1023,0.01635733370979627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,1023,0.023397333920001984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,1023,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,1023,0.019445333629846573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,2047,0.016048000504573185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,2047,0.03377600014209747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,2047,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,2047,0.02957333376010259
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,4095,0.015370666980743408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,4095,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,4095,0.05400000015894572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,4095,0.05004266897837321
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,8191,0.016778666526079178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,8191,0.09524266918500264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,8191,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,float16,16383,0.016805333395799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,8191,0.09227200349171956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,128,1,float16,fp8,16383,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,1,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,float16,16383,0.17893866697947183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,1,0.013855999956528345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,1,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,64,0,1,float16,fp8,16383,0.17105066776275635
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,1,0.010698666175206503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,3,0.014698666830857595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,3,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,7,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,7,0.014192000031471252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,7,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,15,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,7,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,15,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,15,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,31,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,31,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,31,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,63,0.01687466725707054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,63,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,63,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,127,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,63,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,127,0.016330666840076447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,127,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,127,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,255,0.016938666502634685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,255,0.016554666062196095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,255,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,255,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,511,0.015765332927306492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,511,0.017738666385412216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,511,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,511,0.014688000082969666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,1023,0.01594666639963786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,1023,0.023413332800070446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,1023,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,1023,0.01977066695690155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,2047,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,2047,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,2047,0.03380800038576126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,2047,0.029717333614826202
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,4095,0.01634666696190834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,4095,0.054234668612480164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,4095,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,4095,0.05029866596062978
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,8191,0.016965333372354507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,8191,0.0969546635945638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,8191,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,8191,0.09115733702977498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,float16,16383,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,128,1,float16,fp8,16383,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,float16,16383,0.17945599555969238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,64,0,1,float16,fp8,16383,0.1711519956588745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,1,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,1,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,1,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,3,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,7,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,7,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,7,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,7,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,15,0.011957333733638128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,15,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,15,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,31,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,31,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,31,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,63,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,127,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,127,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,127,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,127,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,255,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,255,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,255,0.01202133297920227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,255,0.011877333124478659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,511,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,511,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,511,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,511,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,1023,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,1023,0.012479999413092932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,1023,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,1023,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,2047,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,2047,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,2047,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,2047,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,4095,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,4095,0.01586666703224182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,4095,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,4095,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,8191,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,8191,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,8191,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,8191,0.017498667041460674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,float16,16383,0.012106666962305704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,float16,16383,0.021045332153638203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,128,1,float16,fp8,16383,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,1,0.08911466598510742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,1,0.08891733487447102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,64,0,1,float16,fp8,16383,0.01945066700379054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,1,0.07258666555086772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,1,0.07256533205509186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,3,0.0885706643263499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,3,0.08887466788291931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,3,0.0727893312772115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,3,0.07231466472148895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,7,0.09099733829498291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,7,0.07468800246715546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,7,0.09074133634567261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,7,0.07499200105667114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,15,0.09285866220792134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,15,0.09282132983207703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,15,0.07858666777610779
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,15,0.07869333525498708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,31,0.11125333110491435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,31,0.11158933242162068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,31,0.09733866651852925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,31,0.09730666875839233
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,63,0.11165866255760193
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,63,0.11159466703732808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,63,0.09904533624649048
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,63,0.09910933176676433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,127,0.11334400375684102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,127,0.11316266655921936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,127,0.09927999973297119
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,127,0.0995093286037445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,255,0.11342400312423706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,255,0.1322879989941915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,255,0.09916800260543823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,255,0.11969066659609477
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,511,0.11355732878049214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,511,0.19744000832239786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,511,0.09918399651845296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,511,0.18499734004338583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,1023,0.11365866661071777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,1023,0.09920000036557515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,1023,0.32654933134714764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,1023,0.31431466341018677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,float16,2047,0.11374933520952861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,128,1,float16,fp8,2047,0.09921600421269734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,float16,2047,0.5911466677983602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,1,0.08901866277058919
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,64,0,1,float16,fp8,2047,0.576255997021993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,1,0.08892800410588582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,1,0.07249600191911061
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,1,0.07365333537260692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,3,0.07263466715812683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,3,0.08901333808898926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,3,0.08875733613967896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,7,0.09109866619110107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,7,0.07469866673151652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,7,0.09110400080680847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,3,0.073594664533933
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,7,0.0746559997399648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,15,0.09288533528645833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,15,0.09297066926956177
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,15,0.07926933467388153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,15,0.07865066826343536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,31,0.11129066348075867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,31,0.1116426686445872
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,31,0.09821866949399312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,31,0.09703999757766724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,63,0.11222933729489644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,63,0.11337066690127055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,63,0.09975467125574748
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,63,0.09889599680900574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,127,0.11352533102035522
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,127,0.11357333262761433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,127,0.09903466701507568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,255,0.11364799737930298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,127,0.0999893347422282
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,255,0.1327359974384308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,255,0.09924800197283427
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,255,0.1197706659634908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,511,0.11405866344769795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,511,0.19787200291951498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,511,0.09938666224479675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,511,0.18525334199269614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,1023,0.11399466792742412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,1023,0.09923199812571208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,1023,0.3261973261833191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,1023,0.31436266501744586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,float16,2047,0.11429333686828613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,128,1,float16,fp8,2047,0.09941866993904114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,1,0.08927999933560689
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,1,0.08867200215657552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,float16,2047,0.5909866491953532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,1,0.07253333429495494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,1,0.07261333366235097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,64,0,1,float16,fp8,2047,0.5769173304239908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,3,0.08921600381533305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,3,0.08898133039474487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,3,0.07286400099595387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,3,0.0727040022611618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,7,0.09108799695968628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,7,0.09077333410580952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,7,0.07603733241558075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,7,0.07453333338101704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,15,0.09309867024421692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,15,0.09345600008964539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,15,0.07840000092983246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,15,0.07899199922879536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,31,0.11184533437093098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,31,0.09899733463923137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,31,0.11205333471298218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,31,0.09922666351000468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,63,0.11375466982523601
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,63,0.11196266611417134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,63,0.09913067022959392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,63,0.09963200489679973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,127,0.11369066437085469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,127,0.11353600025177002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,127,0.09977066516876221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,127,0.0995306670665741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,255,0.11401599645614624
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,255,0.13410666584968567
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,255,0.09923733274141948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,255,0.11955733100573222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,511,0.11390399932861328
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,511,0.1979680061340332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,511,0.09959999720255534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,511,0.18552533785502115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,1023,0.1143946647644043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,1023,0.326474666595459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,1023,0.09990933537483215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,1023,0.3150026599566142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,float16,2047,0.11365866661071777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,128,1,float16,fp8,2047,0.09918933113416036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,float16,2047,0.6008266607920328
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,1,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,64,0,1,float16,fp8,2047,0.5762079954147339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,1,0.015583999454975128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,1,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,1,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,3,0.01552533358335495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,3,0.015279999623696009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,3,0.016773333152135212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,3,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,7,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,7,0.015546667079130808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,7,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,7,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,15,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,15,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,15,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,15,0.017114666601022083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,31,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,31,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,31,0.01691199963291486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,63,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,31,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,63,0.015333333363135656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,127,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,63,0.016885332763195038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,63,0.01617066686352094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,127,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,127,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,127,0.01687466725707054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,255,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,255,0.015850666910409927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,255,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,255,0.016762666404247284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,511,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,511,0.01709866647919019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,511,0.018794666975736618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,511,0.01894933357834816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,1023,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,1023,0.025125332176685333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,1023,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,1023,0.023408000667889912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,float16,2047,0.017157333592573803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,128,1,float16,fp8,2047,0.018746666610240936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,float16,2047,0.0420959989229838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,64,0,1,float16,fp8,2047,0.03756266583998998
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,1,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,1,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,1,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,1,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,3,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,3,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,3,0.01488000030318896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,3,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,7,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,7,0.013914667069911957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,7,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,7,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,15,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,15,0.013690666606028875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,15,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,15,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,31,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,31,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,31,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,31,0.016879999389251072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,63,0.014954666296641031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,63,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,63,0.021189334491888683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,63,0.01913600042462349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,127,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,127,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,127,0.0194560003777345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,127,0.02130666623512904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,255,0.014736000448465347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,255,0.016751999656359356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,255,0.020879998803138733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,255,0.021162666380405426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,511,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,511,0.021541332205136616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,511,0.021125334004561108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,511,0.0235359991590182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,1023,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,1023,0.03169599920511246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,1023,0.01941866676012675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,1023,0.03378133227427801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,2047,0.016399999459584553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,2047,0.05231466889381409
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,2047,0.02107200026512146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,2047,0.054005334774653115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,4095,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,4095,0.09538666407267253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,4095,0.021301334102948506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,4095,0.09301333626111348
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,8191,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,8191,0.021007999777793884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,8191,0.17961599429448447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,8191,0.1711519956588745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,float16,16383,0.015333333363135656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,128,1,float16,fp8,16383,0.020997333029905956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,1,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,float16,16383,0.35397334893544513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,1,0.013898666948080063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,1,0.015504000087579092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,1,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,64,0,1,float16,fp8,16383,0.32685333490371704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,3,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,3,0.014090667168299357
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,3,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,3,0.015493333339691162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,7,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,7,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,7,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,7,0.015386667102575302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,15,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,15,0.014170666535695394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,15,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,15,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,31,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,31,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,31,0.017125333348910015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,31,0.01703466723362605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,63,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,63,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,63,0.02165866643190384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,63,0.021130666136741638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,127,0.014954666296641031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,127,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,127,0.021573332448800404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,127,0.021013334393501282
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,255,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,255,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,255,0.021040000021457672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,255,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,511,0.016506666938463848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,511,0.021295999487241108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,511,0.02144533395767212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,511,0.02404266595840454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,1023,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,1023,0.03215466688076655
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,1023,0.021130666136741638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,1023,0.03339199970165888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,2047,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,2047,0.053455998500188194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,2047,0.02111999938885371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,2047,0.05305066704750061
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,4095,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,4095,0.09526399771372478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,4095,0.019621333728233974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,8191,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,4095,0.09417600433031718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,8191,0.1790506641070048
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,8191,0.01926400015751521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,8191,0.17106133699417114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,float16,16383,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,128,1,float16,fp8,16383,0.019386666516462963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,1,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,float16,16383,0.35389868418375653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,1,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,1,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,1,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,64,0,1,float16,fp8,16383,0.3266879916191101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,3,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,3,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,3,0.01532799998919169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,3,0.01532799998919169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,7,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,7,0.014149333039919535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,7,0.015498666713635126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,7,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,15,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,15,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,15,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,31,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,15,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,31,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,31,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,31,0.017184000462293625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,63,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,63,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,63,0.021066665649414062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,63,0.0210506667693456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,127,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,127,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,127,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,127,0.020986666282018025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,255,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,255,0.01706133286158244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,255,0.020938667158285778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,255,0.02120000123977661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,511,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,511,0.020954666038354237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,511,0.021301334102948506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,511,0.023029332359631855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,1023,0.015370666980743408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,1023,0.019381333142518997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,1023,0.031850665807724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,1023,0.03367999941110611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,2047,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,2047,0.05351466437180837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,2047,0.020997333029905956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,2047,0.052613332867622375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,4095,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,4095,0.01939733326435089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,4095,0.09512533744176228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,4095,0.09282666444778442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,8191,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,8191,0.021104000508785248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,8191,0.1811413367589315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,8191,0.17030400037765503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,float16,16383,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,float16,16383,0.3557120164235433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,128,1,float16,fp8,16383,0.01945066700379054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,1,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,64,0,1,float16,fp8,16383,0.32652799288431805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,1,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,3,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,3,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,3,0.012085333466529846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,3,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,7,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,7,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,7,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,7,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,15,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,15,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,15,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,15,0.01221866657336553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,31,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,31,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,31,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,63,0.010591999938090643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,63,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,63,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,127,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,127,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,127,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,127,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,255,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,255,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,255,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,255,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,511,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,511,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,511,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,511,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,1023,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,1023,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,1023,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,1023,0.013978666315476099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,2047,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,2047,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,2047,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,4095,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,2047,0.01598400001724561
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,4095,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,4095,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,4095,0.01748266691962878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,8191,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,8191,0.019658666104078293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,8191,0.019343999524911244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,8191,0.013631999492645264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,float16,16383,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,float16,16383,0.025434667865435284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,128,1,float16,fp8,16383,0.013365333278973898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,64,0,1,float16,fp8,16383,0.025759999950726826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,1,0.1695680022239685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,1,0.17005866765975952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,1,0.13758933544158936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,1,0.13732799887657166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,3,0.17000534137090048
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,3,0.16985599199930826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,3,0.13685866196950278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,3,0.13674666484196982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,7,0.17492800951004028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,7,0.14267733693122864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,7,0.17523199319839478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,7,0.14286933342615762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,15,0.17905600865681967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,15,0.1774239937464396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,15,0.149018665154775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,15,0.14867732922236124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,31,0.21618133783340454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,31,0.18930667638778687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,31,0.2159199913342794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,31,0.18949333826700845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,63,0.21792000532150269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,63,0.21799467007319132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,63,0.18921067317326865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,63,0.19006399313608804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,127,0.22009599208831787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,127,0.21982399622599283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,127,0.19144533077875772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,127,0.1913493275642395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,255,0.22000000874201456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,255,0.2590293288230896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,255,0.1917333404223124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,255,0.23042666912078857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,511,0.22036266326904297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,511,0.19113600254058838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,511,0.3880159854888916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,511,0.361786683400472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,float16,1023,0.22054932514826456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,1,0.168938676516215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,float16,1023,0.6438186566034952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,128,1,float16,fp8,1023,0.19235734144846597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,64,0,1,float16,fp8,1023,0.619983990987142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,1,0.16905067364374796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,1,0.13822933038075766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,1,0.13864533106486002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,3,0.17070400714874268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,3,0.1690346598625183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,3,0.1378933290640513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,3,0.1386186679204305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,7,0.17513600985209146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,7,0.1751733422279358
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,7,0.14235732952753702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,7,0.14412267009417215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,15,0.17909334103266397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,15,0.17731199661890665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,15,0.14841600259145102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,15,0.14891733725865683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,31,0.18905067443847656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,31,0.21619200706481934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,31,0.21568000316619873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,31,0.18910932540893555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,63,0.18952532609303793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,63,0.2179786761601766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,63,0.21836799383163452
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,63,0.1913813352584839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,127,0.22007999817530313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,127,0.19141866763432822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,127,0.22003199656804404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,127,0.1916159987449646
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,255,0.21998933951059976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,255,0.1914400060971578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,255,0.25897600253423053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,255,0.23222933212916055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,511,0.21998933951059976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,511,0.19165333112080893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,511,0.3875733216603597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,511,0.36188801129659015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,float16,1023,0.2202826738357544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,128,1,float16,fp8,1023,0.19150932629903158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,1,0.16910399993260702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,float16,1023,0.6441653172175089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,1,0.16885866721471152
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,1,0.13821867108345032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,64,0,1,float16,fp8,1023,0.6196639935175577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,1,0.138373335202535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,3,0.16920000314712524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,3,0.13823999961217245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,3,0.16915732622146606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,3,0.1381226678689321
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,7,0.17493865887324014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,7,0.1421280006567637
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,7,0.17510932683944702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,7,0.14246933658917746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,15,0.1788640022277832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,15,0.1788426637649536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,15,0.14870400230089822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,15,0.14849066734313965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,31,0.21586666504542032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,31,0.2159199913342794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,31,0.1894986629486084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,31,0.18944533665974936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,63,0.21795199314753214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,63,0.21808000405629477
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,63,0.19124799966812134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,63,0.19113065799077353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,127,0.22013866901397705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,127,0.22001065810521445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,127,0.19164266188939413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,127,0.191429336865743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,255,0.2203786571820577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,255,0.25896533330281574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,255,0.19137599070866904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,255,0.23218133052190146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,511,0.2214453419049581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,511,0.1914986570676168
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,511,0.38601601123809814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,511,0.36165865262349445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,float16,1023,0.22028799851735434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,128,1,float16,fp8,1023,0.19147199392318726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,float16,1023,0.6573706865310669
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,1,0.021205333371957142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,1,0.019050666441520054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,64,0,1,float16,fp8,1023,0.619322657585144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,1,0.021173333128293354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,1,0.01934933289885521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,3,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,3,0.021002667645613354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,3,0.01939733326435089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,3,0.01977066695690155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,7,0.02107200026512146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,7,0.019285333653291065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,7,0.020917333662509918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,7,0.02088533341884613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,15,0.01904533306757609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,15,0.020997333029905956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,15,0.019551999866962433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,15,0.019109333554903667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,31,0.02109866589307785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,31,0.01916266605257988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,31,0.020682666450738907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,31,0.021903999149799347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,63,0.019152000546455383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,63,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,63,0.021125334004561108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,127,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,127,0.019194666296243668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,127,0.02083733429511388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,63,0.020053333292404812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,127,0.020128000527620316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,255,0.01923199991385142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,255,0.021045332153638203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,255,0.02162666618824005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,255,0.0198186660806338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,511,0.02102400114138921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,511,0.026933332284291584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,511,0.025562666356563568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,511,0.021727999051411945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,float16,1023,0.0210506667693456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,float16,1023,0.03972800076007843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,128,1,float16,fp8,1023,0.021141332884629566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,64,0,1,float16,fp8,1023,0.035989334185918175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,1,0.33060266574223834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,1,0.3307360013326009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,1,0.2656746705373128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,1,0.26549333333969116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,3,0.3307093381881714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,3,0.3315466642379761
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,3,0.26525866985321045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,3,0.26552534103393555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,7,0.3410826524098714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,7,0.3418080012003581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,7,0.277621328830719
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,7,0.27794132630030316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,15,0.349018653233846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,15,0.3491733471552531
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,15,0.2898133397102356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,15,0.2900000015894572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,31,0.4231413205464681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,31,0.42271467049916583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,31,0.36963733037312824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,31,0.36984535058339435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,63,0.4270346562067668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,63,0.37353599071502686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,63,0.4271786610285441
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,63,0.37354131539662677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,127,0.43089600404103595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,127,0.37570667266845703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,127,0.4313066800435384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,127,0.37619733810424805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,fp8,255,0.3758560021718343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,128,1,float16,float16,255,0.4329440196355184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,float16,255,0.5091466506322225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,1,0.33079467217127484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,1,0.3309173385302226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,64,0,1,float16,fp8,255,0.4551466703414917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,1,0.26711465915044147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,1,0.26746666431427
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,3,0.3311520020167033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,3,0.3306666612625122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,3,0.2674720088640849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,3,0.2672640085220337
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,7,0.34187201658884686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,7,0.3426293134689331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,7,0.27932800849278766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,7,0.2774239977200826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,15,0.3494186798731486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,15,0.349237322807312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,15,0.28969067335128784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,15,0.2898240089416504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,31,0.42421332995096844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,31,0.4241386651992798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,31,0.3702293237050374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,31,0.3699573278427124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,63,0.4286453326543172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,63,0.4285973310470581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,63,0.3736480077107747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,63,0.37378132343292236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,127,0.4329226811726888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,127,0.43277867635091144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,127,0.3758080005645752
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,float16,255,0.43347732226053876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,float16,255,0.5089653333028158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,127,0.3768959840138753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,128,1,float16,fp8,255,0.3758080005645752
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,1,0.331770658493042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,64,0,1,float16,fp8,255,0.45554665724436444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,1,0.3319946726163228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,1,0.26910400390625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,1,0.26944533983866376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,3,0.33260265986124676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,3,0.269322673479716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,3,0.3330986698468526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,3,0.26951466004053753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,7,0.343120018641154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,7,0.3425653378168742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,7,0.2799253265062968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,7,0.2792053421338399
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,15,0.34935998916625977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,15,0.3494400183359782
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,15,0.29206933577855426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,15,0.2916053334871928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,31,0.42476801077524823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,31,0.42293334007263184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,31,0.37050668398539227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,31,0.37188267707824707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,63,0.4291893243789673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,63,0.4288586775461833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,63,0.3737599849700928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,63,0.37405868371327716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,127,0.4329440196355184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,127,0.3758026758829753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,127,0.4322613477706909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,127,0.37624533971150714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,float16,255,0.4341760079065959
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,128,1,float16,fp8,255,0.3757599989573161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,float16,255,0.510693351427714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,64,0,1,float16,fp8,255,0.45577065149943036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,1,0.030906667311986286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,1,0.029418667157491047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,1,0.031317333380381264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,1,0.031258667508761086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,3,0.030623999734719593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,3,0.029296000798543293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,3,0.03141333411137263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,3,0.0317493329445521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,7,0.030133334298928578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,7,0.029370665550231934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,7,0.031194667021433514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,7,0.031311998764673867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,15,0.02994133283694585
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,15,0.029306667546431225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,15,0.031167998909950256
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,15,0.031114667654037476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,31,0.029605334003766377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,31,0.02958400050799052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,31,0.03120533376932144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,31,0.031285333136717476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,63,0.029535998900731403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,63,0.031370667119820915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,63,0.03127466638882955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,63,0.03166933357715607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,127,0.02992533395687739
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,127,0.02959466725587845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,127,0.031530665854612984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,127,0.03121600051720937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,float16,255,0.03166933357715607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,128,1,float16,fp8,255,0.03192000091075897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,float16,255,0.03036266565322876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,64,0,1,float16,fp8,255,0.031136001149813335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,1,0.5236373345057169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,1,0.6546773513158163
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,1,0.6545279820760092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,1,0.5232479969660441
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,3,0.6544906695683798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,3,0.6549546718597412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,3,0.5232640107472738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,3,0.5239733457565308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,7,0.6769759654998779
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,7,0.6772106488545736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,7,0.5484319925308228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,7,0.5489120086034139
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,15,0.6915520032246908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,15,0.6912213166554769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,15,0.5726826588312784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,15,0.5739839871724447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,31,0.8394133249918619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,31,0.8395040035247803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,31,0.732431968053182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,31,0.7322026888529459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,63,0.848848025004069
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,63,0.739904006322225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,63,0.8491040070851644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,63,0.7385173638661703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,float16,127,0.8568906784057617
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,float16,127,0.857701301574707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,0,1,float16,fp8,127,0.7441653410593668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,64,128,1,float16,fp8,127,0.7444586753845215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,1,0.6546080112457275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,1,0.6552213430404663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,1,0.527237335840861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,1,0.526469349861145
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,3,0.6547040144602457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,3,0.6557439963022867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,3,0.5272800127665201
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,3,0.526469349861145
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,7,0.6770346959431967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,7,0.6770986715952555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,7,0.5494346618652344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,7,0.5499733289082845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,15,0.6916639804840088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,15,0.6916213035583496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,15,0.5741066535313925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,15,0.5739359855651855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,31,0.8410720030466715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,31,0.7328106562296549
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,31,0.8406986395517985
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,31,0.7334933280944824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,63,0.8507680098215739
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,63,0.7396106719970703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,63,0.8496479988098145
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,63,0.7402880191802979
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,float16,127,0.8584106763203939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,128,1,float16,fp8,127,0.7445387045542399
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,float16,127,0.8586666584014893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,64,0,1,float16,fp8,127,0.7450933456420898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,1,0.6561280091603597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,1,0.530842661857605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,1,0.6559466520945231
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,1,0.5311199824015299
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,3,0.6569066842397054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,3,0.5306453307469686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,3,0.6557493209838867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,3,0.531823992729187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,7,0.6767040093739828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,7,0.5503520170847574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,7,0.6770400206247965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,7,0.5501919984817505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,15,0.6910719871520996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,15,0.5740106503168741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,15,0.574341336886088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,15,0.6915199756622314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,31,0.8409173488616943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,31,0.8401386737823486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,31,0.7338933149973551
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,31,0.7338399887084961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,63,0.7396533489227295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,63,0.740511973698934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,63,0.850704034169515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,63,0.850005308787028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,fp8,127,0.7453013261159261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,128,1,float16,float16,127,0.857647975285848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,fp8,127,0.7447466850280762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,64,0,1,float16,float16,127,0.8582719961802164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,1,0.05005866785844167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,1,0.05040533343950907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,1,0.05002133548259735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,1,0.050250664353370667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,3,0.050293331344922386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,3,0.04980800052483877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,3,0.04996266464392344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,3,0.05202666421731313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,7,0.05039466420809428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,7,0.05003199974695841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,7,0.05019199848175049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,7,0.05190399785836538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,15,0.05020266771316528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,15,0.04987200101216634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,15,0.05193066596984863
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,15,0.05193066596984863
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,31,0.04994133114814758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,31,0.05016533533732096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,31,0.05133333305517832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,31,0.05008533100287119
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,63,0.049685334165891014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,63,0.05031466484069824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,63,0.05060266455014547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,63,0.051216001311937966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,float16,127,0.05118933320045471
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,float16,127,0.05012799799442291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,128,1,float16,fp8,127,0.050741334756215416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,64,0,1,float16,fp8,127,0.051642666260401406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,1,0.018277333428462345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,1,0.018933333456516266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,1,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,1,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,3,0.01915733392039935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,3,0.018794666975736618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,3,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,3,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,7,0.019285333653291065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,7,0.018965333700180054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,7,0.01482133318980535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,7,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,15,0.01850133389234543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,15,0.018911999960740406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,15,0.015477333217859268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,15,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,31,0.019280000279347103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,31,0.02143999934196472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,31,0.018768000106016796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,31,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,63,0.020469332734743755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,63,0.02089066555102666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,63,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,63,0.01727466657757759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,127,0.02111999938885371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,127,0.019066666563351948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,127,0.01836799954374631
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,127,0.01884799947341283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,255,0.019280000279347103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,255,0.022991999983787537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,255,0.01899733394384384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,255,0.021125334004561108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,511,0.020954666038354237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,511,0.03146666785081228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,511,0.017093333105246227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,511,0.029509333272775013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,1023,0.020938667158285778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,1023,0.0183146670460701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,1023,0.04976533353328705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,1023,0.04776533444722494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,2047,0.020981334149837494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,2047,0.0848586658636729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,2047,0.018698666244745255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,2047,0.0827893316745758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,4095,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,4095,0.018160000443458557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,4095,0.15637866655985513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,4095,0.1544319987297058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,8191,0.02110933264096578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,8191,0.017344000438849132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,8191,0.2985440095265706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,8191,0.29686933755874634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,float16,16383,0.021018666525681812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,1,0.0183146670460701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,1,0.019093333433071773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,128,1,float16,fp8,16383,0.01937066639463107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,1,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,float16,16383,0.6177173455556234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,3,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,1,0.015840000162522
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,3,0.019071999937295914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,3,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,64,0,1,float16,fp8,16383,0.6237066586812338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,3,0.015834666788578033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,7,0.01929066702723503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,7,0.019120000302791595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,7,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,15,0.019152000546455383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,7,0.015504000087579092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,15,0.01926400015751521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,15,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,15,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,31,0.019498666127522785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,31,0.02146133283774058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,31,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,31,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,63,0.02035733312368393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,63,0.018922666708628338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,63,0.021344001094500225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,63,0.01695466662446658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,127,0.02111999938885371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,127,0.020992000897725422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,127,0.01926400015751521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,127,0.018496000518401463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,255,0.02110933264096578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,255,0.02330133318901062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,255,0.021104000508785248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,255,0.019333332777023315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,511,0.021157334248224895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,511,0.03163733333349228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,511,0.01889066646496455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,511,0.029557332396507263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,1023,0.021541332205136616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,1023,0.04966933528582255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,1023,0.01912533367673556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,1023,0.04789866507053375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,2047,0.021136000752449036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,2047,0.08483200271924336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,2047,0.019061333189407986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,2047,0.08313600222269694
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,4095,0.021002667645613354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,4095,0.15648000439008078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,4095,0.01883200059334437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,4095,0.15475199619928995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,8191,0.021125334004561108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,8191,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,8191,0.29950932661692303
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,8191,0.2979840040206909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,float16,16383,0.021029333273569744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,128,1,float16,fp8,16383,0.018719999740521114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,1,0.018789333601792652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,float16,16383,0.6171893278757731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,1,0.019088000059127808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,1,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,1,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,3,0.018837332725524902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,64,0,1,float16,fp8,16383,0.6249173482259115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,3,0.019178666174411774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,3,0.015306666493415833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,3,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,7,0.019093333433071773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,7,0.0191040001809597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,7,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,7,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,15,0.01889066646496455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,15,0.01894933357834816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,15,0.015439999600251516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,15,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,31,0.01939733326435089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,31,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,31,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,31,0.01710933322707812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,63,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,63,0.020527999848127365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,63,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,63,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,127,0.020938667158285778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,127,0.019120000302791595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,127,0.018810667097568512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,127,0.01877333347996076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,255,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,255,0.02314666658639908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,255,0.01918399954835574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,255,0.02096533278624217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,511,0.02103466788927714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,511,0.03162666658560435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,511,0.01704000060757001
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,511,0.031290667752424874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,1023,0.020960000654061634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,1023,0.05008000135421753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,1023,0.019061333189407986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,1023,0.048154667019844055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,2047,0.02102400114138921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,2047,0.08504533767700195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,2047,0.01904533306757609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,2047,0.08269333342711131
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,4095,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,4095,0.01923199991385142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,4095,0.1566933294137319
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,4095,0.1546933352947235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,8191,0.021290667355060577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,8191,0.30001600583394367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,8191,0.01874133323629697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,8191,0.2980213363965352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,float16,16383,0.021546666820844013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,128,1,float16,fp8,16383,0.019088000059127808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,1,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,1,0.011765333513418833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,1,0.011695999652147293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,float16,16383,0.6382879813512167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,3,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,3,0.01163200040658315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,64,0,1,float16,fp8,16383,0.6260906855265299
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,3,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,7,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,7,0.011957333733638128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,7,0.012565333396196365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,15,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,15,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,15,0.012223999947309494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,15,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,31,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,31,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,31,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,63,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,63,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,63,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,63,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,127,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,127,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,127,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,127,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,255,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,255,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,255,0.012693333129088083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,255,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,511,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,511,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,511,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,511,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,1023,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,1023,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,1023,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,2047,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,2047,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,2047,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,2047,0.016976000120242436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,4095,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,4095,0.020938667158285778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,4095,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,4095,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,8191,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,8191,0.027093333502610523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,8191,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,8191,0.025221332907676697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,float16,16383,0.013584000368913015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,float16,16383,0.04182933270931244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,128,1,float16,fp8,16383,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,64,0,1,float16,fp8,16383,0.035487999518712364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,1,1.300544023513794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,1,1.0359840393066406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,1,1.299397309621175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,1,1.0371519724527996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,3,1.301029364267985
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,3,1.2999200026194255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,3,1.0378026962280273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,3,1.0384746392567952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,7,1.3452107111612956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,7,1.3461012840270996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,7,1.0908106962839763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,7,1.0915306409200032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,15,1.374010721842448
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,15,1.375088055928548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,15,1.1397973696390789
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,15,1.140496015548706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,31,1.6714986165364583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,31,1.673141320546468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,31,1.4588054021199544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,31,1.4582133293151855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,float16,63,1.6900320053100586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,float16,63,1.6916534105936687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,128,1,float16,fp8,63,1.4718720118204753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,64,0,1,float16,fp8,63,1.4721226692199707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,1,1.300976037979126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,1,1.3010613123575847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,1,1.0456266403198242
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,1,1.0470399856567383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,3,1.3009119828542073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,3,1.3006133238474529
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,3,1.0458026727040608
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,3,1.045738697052002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,7,1.0913173357645671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,7,1.3447359402974446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,7,1.3458773295084636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,7,1.0920053323109944
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,15,1.1396106878916423
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,15,1.374853293100993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,15,1.3738719622294109
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,15,1.1401279767354329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,31,1.4582719802856445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,31,1.6735199292500813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,31,1.6733226776123047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,31,1.4585493405659993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,fp8,63,1.471984068552653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,128,1,float16,float16,63,1.691914717356364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,float16,63,1.6932586034138997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,64,0,1,float16,fp8,63,1.4720533688863118
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,1,1.0544906457265217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,1,1.3040800094604492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,1,1.3033706347147624
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,1,1.0558666388193767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,3,1.0543146928151448
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,3,1.304469347000122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,3,1.3044053713480632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,3,1.0567200183868408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,7,1.0928853352864583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,7,1.3463306427001953
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,7,1.346981366475423
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,7,1.093237320582072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,15,1.3745813369750977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,15,1.141151984532674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,15,1.3738239606221516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,15,1.1416213512420654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,31,1.4599092801411946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,31,1.6756319999694824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,31,1.6751200358072917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,31,1.459664026896159
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,float16,63,1.703978697458903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,float16,63,1.6996852556864421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,1,0.09090666969617207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,128,1,float16,fp8,63,1.472991943359375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,1,0.09084266424179077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,64,0,1,float16,fp8,63,1.4730933507283528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,1,0.09211200475692749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,1,0.09044800202051799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,3,0.09114666779836018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,3,0.09085333347320557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,3,0.09115733702977498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,7,0.09076799949010213
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,7,0.09121599793434143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,3,0.09051733215649922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,7,0.09063466389973958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,7,0.09039466579755147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,15,0.09107733766237895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,15,0.09130133191744487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,15,0.09116799632708232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,15,0.09109866619110107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,31,0.09106133381525676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,31,0.09139733513196309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,31,0.09098666906356812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,31,0.09096533060073853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,float16,63,0.09105599919954936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,float16,63,0.09125866492589314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,128,1,float16,fp8,63,0.09095999598503113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,64,0,1,float16,fp8,63,0.09134399890899658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,1,2.5967626571655273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,1,2.0696800549825034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,1,2.069546699523926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,1,2.6009066899617515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,3,2.0712265968322754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,3,2.604560057322184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,3,2.073023955027262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,3,2.594154675801595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,7,2.1789493560791016
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,7,2.1744426091512046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,7,2.6994187037150064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,7,2.6903041203816733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,15,2.273183981577555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,15,2.2757439613342285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,15,2.7569494247436523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,15,2.7498292922973633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,fp8,31,2.9158827463785806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,fp8,31,2.9131199518839517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,128,1,float16,float16,31,3.3508691787719727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,64,0,1,float16,float16,31,3.3510026931762695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,1,2.0860586166381836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,1,2.0887200037638345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,1,2.59770139058431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,1,2.596106688181559
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,3,2.09281063079834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,3,2.5988426208496094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,3,2.092485268910726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,3,2.6069812774658203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,7,2.1832106908162436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,7,2.1795573234558105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,7,2.69922669728597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,7,2.6968053181966147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,15,2.2795093854268393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,15,2.2793386777242026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,15,2.7864907582600913
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,15,2.77838929494222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,fp8,31,2.925514539082845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,fp8,31,2.9199094772338867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,0,1,float16,float16,31,3.387200037638346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,64,128,1,float16,float16,31,3.386693318684896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,1,2.1087892850240073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,1,2.621829350789388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,1,2.611087958017985
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,3,2.11301326751709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,3,2.6176160176595054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,3,2.637056032816569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,1,2.110922654469808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,3,2.112490653991699
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,7,2.7336107889811196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,7,2.187397321065267
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,7,2.1904373168945312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,15,2.8860479990641275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,15,2.8875732421875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,7,2.756922721862793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,15,2.308762709299723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,15,2.304405371348063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,float16,31,3.4663572311401367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,128,1,float16,fp8,31,2.9667838414510093
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,float16,31,3.461653391520182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,64,0,1,float16,fp8,31,2.965674718221029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,1,0.16902933518091837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,1,0.16920000314712524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,1,0.17269867658615112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,1,0.1736639936765035
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,3,0.16911999384562174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,3,0.16896533966064453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,3,0.17219199736913046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,3,0.17262399196624756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,7,0.1693440079689026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,7,0.16910399993260702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,7,0.17216533422470093
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,15,0.16965866088867188
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,15,0.16885334253311157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,15,0.17231466372807822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,15,0.1723466714223226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,float16,31,0.1693920095761617
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,float16,31,0.17037334044774374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,1,0.02756800005833308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,1,0.02756800005833308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,1,0.024186665813128155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,1,0.024533333877722423
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,3,0.027488000690937042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,31,0.17299733559290567
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,3,0.027349332968393963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,3,0.023285334308942158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,3,0.02325333406527837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,128,1,float16,fp8,7,0.17394665877024332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,7,0.029317334294319153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,7,0.02770666778087616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,7,0.025285333395004272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,15,0.028064000109831493
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,7,0.025759999950726826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,15,0.029343999922275543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,15,0.025221332907676697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,15,0.02609066665172577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,31,0.03364799916744232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,31,0.03365333378314972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,64,0,1,float16,fp8,31,0.17239999771118164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,31,0.029461334149042766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,31,0.030224000414212544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,63,0.03332799921433131
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,63,0.02976000060637792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,63,0.0313226655125618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,63,0.03377600014209747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,127,0.03364266703526179
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,127,0.03382399926582972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,127,0.0315733328461647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,127,0.02979733298222224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,255,0.033717334270477295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,255,0.039861333866914116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,511,0.03385599950949351
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,255,0.031040000418821972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,255,0.0354720006386439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,511,0.05602133274078369
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,511,0.02959466725587845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,511,0.0536106675863266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,1023,0.03385066737731298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,1023,0.030048000315825146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,1023,0.09102400143941243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,1023,0.08702400326728821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,2047,0.03378133227427801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,2047,0.03146666785081228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,2047,0.16059733430544534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,4095,0.03383466601371765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,2047,0.15652267138163248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,4095,0.029968000948429108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,float16,8191,0.033488000432650246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,4095,0.2996906638145447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,4095,0.29579200347264606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,1,0.0277813325325648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,128,1,float16,fp8,8191,0.03124266614516576
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,1,0.0273333340883255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,1,0.023973333338896435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,1,0.024005333582560223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,float16,8191,0.5784746805826823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,3,0.027797333896160126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,3,0.0277813325325648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,64,0,1,float16,fp8,8191,0.572650671005249
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,3,0.023381332556406658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,3,0.0233599990606308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,7,0.0295413335164388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,7,0.02922133356332779
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,7,0.0252960001428922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,7,0.025045332809289295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,15,0.029343999922275543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,15,0.025744001070658367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,15,0.029824001093705494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,15,0.025616000096003216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,31,0.03365333378314972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,31,0.03352533280849457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,31,0.03002133220434189
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,31,0.029472000896930695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,63,0.03365333378314972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,63,0.033530667424201965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,63,0.031008000175158184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,63,0.029663999875386555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,127,0.03402133285999298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,127,0.033439998825391136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,127,0.029663999875386555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,127,0.03143466760714849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,255,0.03377600014209747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,255,0.03774400055408478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,255,0.029722665747006733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,255,0.036015999813874565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,511,0.03382933388153712
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,511,0.055871998270352684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,511,0.029648000995318096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,511,0.05377600093682607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,1023,0.0337119996547699
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,1023,0.03107733279466629
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,1023,0.09046933054924011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,1023,0.08706133564313252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,2047,0.03363733241955439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,2047,0.15941333770751953
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,2047,0.03143999973932902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,2047,0.1568106710910797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,4095,0.03336533407370249
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,4095,0.29917333523432416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,4095,0.03163733333349228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,4095,0.2946079969406128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,float16,8191,0.03402666747570038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,128,1,float16,fp8,8191,0.029616000751654308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,1,0.02757866680622101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,float16,8191,0.5784480174382528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,1,0.027274665733178455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,1,0.025216000775496166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,64,0,1,float16,fp8,8191,0.572378675142924
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,1,0.02310933421055476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,3,0.027621333797772724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,3,0.02717866748571396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,3,0.023557332654794056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,3,0.02442666639884313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,7,0.02769600103298823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,7,0.029194665451844532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,7,0.02534399926662445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,15,0.029322666426499683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,7,0.024501333634058636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,15,0.029109333952267964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,15,0.02537599951028824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,15,0.025045332809289295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,31,0.03365333378314972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,31,0.033743999898433685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,31,0.029391999046007793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,31,0.02977599948644638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,63,0.03350933392842611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,63,0.03367999941110611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,63,0.029616000751654308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,63,0.030720000465710957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,127,0.03341866781314214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,127,0.03365333378314972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,127,0.0314026673634847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,127,0.02998399982849757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,255,0.03370666752258936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,255,0.029637334247430164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,255,0.03956799954175949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,255,0.035631999373435974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,511,0.03342933456103007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,511,0.05625066657861074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,511,0.029909332593282063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,511,0.052255998055140175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,1023,0.033589333295822144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,1023,0.09037866195042928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,1023,0.03124266614516576
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,1023,0.08702400326728821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,2047,0.03368533402681351
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,2047,0.16056000192960104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,2047,0.03148799886306127
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,2047,0.15690132975578308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,4095,0.033333333830038704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,4095,0.29993067185084027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,4095,0.03013866643110911
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,4095,0.2941333254178365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,float16,8191,0.033610666791598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,128,1,float16,fp8,8191,0.02962133288383484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,1,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,float16,8191,0.5869813362757365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,1,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,1,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,3,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,64,0,1,float16,fp8,8191,0.5724213520685831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,3,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,1,0.013594667116800943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,3,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,7,0.012058666596810022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,7,0.011776000261306763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,7,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,15,0.012223999947309494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,15,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,15,0.011962667107582092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,31,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,31,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,63,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,31,0.011855999628702799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,63,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,63,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,127,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,127,0.011765333513418833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,127,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,255,0.012005332857370377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,255,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,255,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,255,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,511,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,511,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,511,0.011877333124478659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,511,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,1023,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,1023,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,1023,0.011957333733638128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,1023,0.014896000425020853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,2047,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,2047,0.01897066707412402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,2047,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,2047,0.018725333114465077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,4095,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,4095,0.025674665967623394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,4095,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,4095,0.02351466566324234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,float16,8191,0.01349866638580958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,float16,8191,0.042064001162846885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,128,1,float16,fp8,8191,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,64,0,1,float16,fp8,8191,0.033701332906881966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,1,0.045850664377212524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,1,0.03972266614437103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,3,0.04779199759165446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,3,0.041333332657814026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,7,0.04808533191680908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,7,0.04215999941031138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,15,0.05180266499519348
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,15,0.058176000912984215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,31,0.0580266664425532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,31,0.052095999320348106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,63,0.0580266664425532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,63,0.052517334620157875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,127,0.06402666866779327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,127,0.06842133402824402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,255,0.10126933455467224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,255,0.09521067142486572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,511,0.16683199008305868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,511,0.16065067052841187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,1023,0.2991146643956502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,1023,0.2863253355026245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,2047,0.5624266862869263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,2047,0.541919986406962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,1,0.04585599899291992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,1,0.03969600051641464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,float16,4095,1.0930293401082356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,3,0.047882666190465294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,3,0.04154133299986521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,7,0.048026666045188904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,1,128,0,1,float16,fp8,4095,1.0518720149993896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,7,0.04181333382924398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,15,0.05797866483529409
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,15,0.051829333106676735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,31,0.05801600217819214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,31,0.0518453319867452
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,63,0.058186665177345276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,127,0.06271466612815857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,127,0.06837333242098491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,255,0.10111467043558757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,255,0.09546666344006856
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,511,0.16667733589808145
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,511,0.1601653297742208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,1023,0.299344003200531
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,1023,0.2871893246968587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,63,0.05266666909058889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,2047,0.5622773170471191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,2047,0.5419413248697916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,1,0.045935998360315956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,1,0.03957866628964742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,3,0.047685335079828896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,3,0.04162133236726125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,7,0.04811733464399973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,7,0.04186133543650309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,float16,4095,1.1048800150553386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,15,0.058101331194241844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,2,128,0,1,float16,fp8,4095,1.052517334620158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,15,0.052389333645502724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,31,0.05816533168156942
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,31,0.05249600112438202
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,63,0.05190933247407278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,63,0.0588319996992747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,127,0.06832000116507213
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,127,0.06284800171852112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,255,0.1011840005715688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,255,0.09542933106422424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,511,0.1604320009549459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,511,0.16681599617004395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,1023,0.2983520030975342
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,1023,0.28762133916219074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,2047,0.5429973204930624
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,2047,0.5672426621119181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,1,0.014773332824309668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,1,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,3,0.01482133318980535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,3,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,7,0.014762666076421738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,7,0.013823999712864557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,15,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,15,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,float16,4095,1.1159199873606365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,4,128,0,1,float16,fp8,4095,1.0589386622111003
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,31,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,31,0.014773332824309668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,63,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,63,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,127,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,127,0.015423999478419622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,255,0.01463466634353002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,255,0.01482133318980535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,511,0.01580799991885821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,511,0.017077332983414333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,1023,0.020010666300853092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,1023,0.019215999792019527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,2047,0.03973866750796636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,2047,0.02731200059254964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,float16,4095,0.06085333228111267
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,1,0.009423999736706415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,128,8,128,0,1,float16,fp8,4095,0.04357333481311798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,1,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,3,0.009445333232482275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,7,0.009152000149091085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,3,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,15,0.009375999992092451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,15,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,63,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,127,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,127,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,255,0.021583999196688335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,255,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,511,0.023391999304294586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,511,0.02757866680622101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,1023,0.031370667119820915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,1023,0.03198933353026708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,2047,0.0420959989229838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,2047,0.041989331444104515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,4095,0.06446399788061778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,4095,0.06242666641871134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,8191,0.11165866255760193
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,8191,0.09955733021100362
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,1,0.00878399983048439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,float16,16383,0.20379199584325156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,1,128,0,1,float16,fp8,16383,0.1769226590792338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,3,0.009461333354314169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,3,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,7,0.009253333633144697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,7,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,15,0.00922133338948091
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,31,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,31,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,63,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,63,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,255,0.021290667355060577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,127,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,255,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,511,0.02492800106604894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,511,0.02789866675933202
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,1023,0.03036266565322876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,1023,0.03192000091075897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,2047,0.04189866781234741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,2047,0.042170668641726174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,4095,0.0661653329928716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,4095,0.06190933287143707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,8191,0.11192533373832703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,8191,0.1011306643486023
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,1,0.009312000125646591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,1,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,float16,16383,0.2042186657587687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,2,128,0,1,float16,fp8,16383,0.17734400431315103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,3,0.009189333145817121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,7,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,7,0.009610666582981745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,15,0.009002666920423508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,31,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,63,0.010501333822806677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,63,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,127,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,127,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,255,0.021520001192887623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,255,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,511,0.024165332317352295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,511,0.027690666417280834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,1023,0.029461334149042766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,2047,0.04197333256403605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,1023,0.031770666440327965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,2047,0.04286933441956838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,4095,0.06414933502674103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,4095,0.0621919979651769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,8191,0.11159466703732808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,8191,0.10089600086212158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,1,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,1,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,float16,16383,0.20367467403411865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,4,128,0,1,float16,fp8,16383,0.1767680048942566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,3,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,3,0.011946666985750198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,7,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,15,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,15,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,31,0.010725333044926325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,31,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,63,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,63,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,127,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,127,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,255,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,511,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,511,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,1023,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,1023,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,2047,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,2047,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,4095,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,4095,0.01727466657757759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,8191,0.018613333503405254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,8191,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,1,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,float16,16383,0.021509334444999695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,128,8,128,0,1,float16,fp8,16383,0.02117866774400075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,1,0.009359999870260557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,3,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,7,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,15,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,31,0.01551466683546702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,31,0.013647999614477158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,63,0.015557333827018738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,63,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,127,0.015861333658297855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,127,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,255,0.017711999515692394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,255,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,511,0.023285334308942158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,511,0.019354666272799175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,1023,0.029696000119050343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,1023,0.033258666594823204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,2047,0.054245332876841225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,2047,0.050186668833096824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,4095,0.09521599610646565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,4095,0.08896533648173015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,8191,0.17764800786972046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,8191,0.16874132553736368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,1,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,float16,16383,0.34301332632700604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,1,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,3,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,1,128,0,1,float16,fp8,16383,0.3267520070075989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,7,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,7,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,15,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,31,0.015466666469971338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,63,0.015333333363135656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,31,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,63,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,127,0.015573333948850632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,127,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,255,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,255,0.017802666872739792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,511,0.023103999594847362
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,511,0.019610666980346043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,1023,0.03409600009520849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,1023,0.029498666524887085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,2047,0.05428266525268555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,2047,0.04997866849104563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,4095,0.09544000029563904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,4095,0.08918399612108867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,8191,0.17944000164667764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,8191,0.16896533966064453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,1,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,1,0.00915733352303505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,3,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,3,0.010608000059922537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,float16,16383,0.34298133850097656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,7,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,7,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,15,0.014778666198253632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,2,128,0,1,float16,fp8,16383,0.32661332686742145
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,15,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,31,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,31,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,63,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,63,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,127,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,127,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,255,0.017610666652520496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,255,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,511,0.022917332748572033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,511,0.01913600042462349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,1023,0.03375466664632162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,1023,0.030069333811601002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,2047,0.05435200035572052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,2047,0.050330668687820435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,4095,0.095551997423172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,4095,0.08924800157546997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,8191,0.1794933279355367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,8191,0.16930667559305826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,1,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,float16,16383,0.34437867005666095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,3,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,1,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,4,128,0,1,float16,fp8,16383,0.33266133069992065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,3,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,7,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,7,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,15,0.01179733375708262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,15,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,31,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,127,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,127,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,255,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,255,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,511,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,511,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,1023,0.012080000092585882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,1023,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,2047,0.015589332828919092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,2047,0.015439999600251516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,4095,0.017290666699409485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,4095,0.017514667163292568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,8191,0.019029332945744198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,8191,0.019648000597953796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,float16,16383,0.023221333821614582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,1,0.08707732955614726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,128,8,128,0,1,float16,fp8,16383,0.021957332889238994
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,1,0.07263466715812683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,3,0.08906132976214091
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,3,0.07643733421961467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,7,0.09093333284060161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,7,0.07881066699822743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,15,0.10969066619873047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,15,0.09719467163085938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,31,0.10940266648928325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,31,0.09899200002352397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,63,0.0993173321088155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,63,0.10936533411343892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,127,0.13009599844614664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,127,0.11967466274897258
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,255,0.19402132431666055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,255,0.18317866325378418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,511,0.3246026635169983
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,511,0.3104906678199768
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,1023,0.5884906848271688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,1023,0.5627626578013102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,1,0.07471466561158498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,1,0.08753066261609395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,3,0.08913066983222961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,float16,2047,1.1129653453826904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,3,0.07727466523647308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,1,128,0,1,float16,fp8,2047,1.0676213105519612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,7,0.09118400017420451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,7,0.07874133189519246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,15,0.11065066854159038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,15,0.0993173321088155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,31,0.11005866527557373
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,31,0.09888000289599101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,63,0.09921600421269734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,127,0.13011200229326883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,63,0.11166933178901672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,127,0.11970667044321696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,255,0.19564799467722574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,255,0.1834239959716797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,511,0.3101653258005778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,511,0.324672003587087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,1023,0.5885759989420573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,1023,0.5616906483968099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,1,0.08693866928418477
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,1,0.07442666590213776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,3,0.07647466659545898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,3,0.08913066983222961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,float16,2047,1.123301347096761
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,2,128,0,1,float16,fp8,2047,1.0676853656768799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,7,0.07890133559703827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,7,0.09116799632708232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,15,0.10965866843859355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,15,0.09930133819580078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,31,0.10980799794197083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,31,0.09895466764767964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,63,0.11117333173751831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,63,0.09948800007502238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,127,0.13012267152468363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,127,0.11983467141787212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,255,0.19556800524393717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,255,0.18339733282725015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,511,0.3247680068016052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,511,0.31014400720596313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,1023,0.5976959864298502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,1023,0.562719980875651
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,1,0.019167999426523846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,float16,2047,1.13481601079305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,1,0.017173333714405697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,3,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,3,0.01693333312869072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,7,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,4,128,0,1,float16,fp8,2047,1.0815040270487468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,7,0.017093333105246227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,15,0.016986666868130367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,15,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,31,0.01747200017174085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,31,0.0174346665541331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,63,0.017397332936525345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,63,0.01786133274435997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,127,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,127,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,255,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,255,0.017658667018016178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,511,0.021359999974568684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,511,0.02143466720978419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,1023,0.03741333385308584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,1023,0.027066667874654133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,float16,2047,0.061477333307266235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,1,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,128,8,128,0,1,float16,fp8,2047,0.044405331214269005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,1,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,3,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,3,0.015685333559910457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,7,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,7,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,15,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,15,0.014490666488806406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,31,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,31,0.021231998999913532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,63,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,63,0.019445333629846573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,127,0.01691199963291486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,127,0.0205226664741834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,255,0.02139200021823247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,255,0.023221333821614582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,511,0.03189333279927572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,511,0.03375466664632162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,1023,0.0524586687485377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,1023,0.05342400074005127
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,2047,0.09192533294359843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,2047,0.09539733330408733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,4095,0.17842666308085123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,4095,0.16885334253311157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,8191,0.3456266721089681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,8191,0.3229653239250183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,1,0.013482666263977686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,1,0.015546667079130808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,float16,16383,0.6904213428497314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,3,0.013418667018413544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,3,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,7,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,15,0.014842666685581207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,7,0.01532799998919169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,1,128,0,1,float16,fp8,16383,0.6303893327713013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,15,0.016943999876578648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,31,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,63,0.014778666198253632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,31,0.01893866683046023
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,63,0.01971199984351794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,127,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,127,0.021087999145189922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,255,0.023381332556406658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,255,0.021114667256673176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,511,0.03142400085926056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,511,0.03344533344109853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,1023,0.05346133311589559
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,1023,0.05384533107280731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,2047,0.09528000156084697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,2047,0.09116266171137492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,4095,0.1792479952176412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,4095,0.16909867525100708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,8191,0.34907734394073486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,8191,0.322437326113383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,1,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,1,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,3,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,3,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,7,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,7,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,float16,16383,0.6996106306711832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,2,128,0,1,float16,fp8,16383,0.6299039920171102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,15,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,15,0.01693333312869072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,31,0.014783999572197596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,63,0.014837333311637243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,63,0.0191040001809597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,31,0.020197333147128422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,127,0.016986666868130367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,127,0.020799999435742695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,255,0.021087999145189922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,255,0.023973333338896435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,511,0.03344533344109853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,511,0.03198933353026708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,1023,0.0521066685517629
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,1023,0.05398933092753092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,2047,0.09517332911491394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,2047,0.09111467003822327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,4095,0.17957866191864014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,4095,0.1692906618118286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,8191,0.3493066628774007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,8191,0.32357333103815716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,1,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,float16,16383,0.7050560315450033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,3,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,4,128,0,1,float16,fp8,16383,0.6321333249409994
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,3,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,7,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,7,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,15,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,15,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,31,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,31,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,63,0.012319999436537424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,127,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,127,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,255,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,255,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,511,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,511,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,1023,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,1023,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,2047,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,2047,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,4095,0.017722666263580322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,4095,0.0174346665541331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,8191,0.02314666658639908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,8191,0.021183999876181286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,float16,16383,0.039690665900707245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,128,8,128,0,1,float16,fp8,16383,0.027589333554108936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,1,0.1669279932975769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,1,0.1400266687075297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,3,0.1715946594874064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,3,0.14436266819636026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,7,0.17517866690953574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,7,0.15040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,15,0.21377599239349365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,15,0.18946133057276407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,31,0.2119200030962626
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,31,0.18978132804234824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,63,0.2137920061747233
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,63,0.18965333700180054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,127,0.23060266176859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,127,0.25305600961049396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,255,0.38225066661834717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,255,0.3577760060628255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,511,0.6408586502075195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,1,0.16686399777730307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,511,0.611135999361674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,1,0.13864533106486002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,3,0.17094399531682333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,fp8,1023,1.1106293201446533
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,3,0.14415466785430908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,7,0.17524266242980957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,1,128,0,1,float16,float16,1023,1.1674933433532715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,7,0.15134933590888977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,15,0.2127679983774821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,15,0.18919465939203897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,31,0.18968000014623007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,31,0.2137653430302938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,63,0.21426665782928467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,63,0.18986133734385172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,127,0.25256532430648804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,127,0.23173866669336954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,255,0.35920000076293945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,255,0.38178666432698566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,511,0.6111520131429037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,511,0.6415733496348063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,1,0.13991999626159668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,1,0.16672533750534058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,3,0.14475733041763306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,3,0.17293866475423178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,fp8,1023,1.1102186838785808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,2,128,0,1,float16,float16,1023,1.177850643793742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,7,0.17539199193318686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,7,0.15245866775512695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,15,0.2130933403968811
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,15,0.1894986629486084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,31,0.1895893414815267
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,31,0.21329599618911743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,63,0.21381866931915283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,63,0.19060266017913818
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,127,0.23224000136057535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,127,0.2529386679331462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,255,0.3825920025507609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,255,0.3593386809031169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,511,0.6114933490753174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,511,0.6555786530176798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,1,0.02367466688156128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,1,0.023760000864664715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,3,0.023760000864664715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,fp8,1023,1.1337440013885498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,4,128,0,1,float16,float16,1023,1.1877386569976807
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,3,0.02330133318901062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,7,0.02325333406527837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,7,0.023552000522613525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,15,0.023344000180562336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,15,0.023578666150569916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,31,0.02346666653951009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,31,0.023765332996845245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,63,0.023215999205907185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,63,0.023567999402681988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,127,0.02364266663789749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,127,0.02317333221435547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,255,0.023599999646345775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,255,0.02333866556485494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,511,0.037765334049860634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,511,0.02977599948644638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,float16,1023,0.05806399881839752
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,128,8,128,0,1,float16,fp8,1023,0.04188799858093262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,1,0.32473599910736084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,1,0.26986666520436603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,3,0.33695467313130695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,3,0.2811466654141744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,7,0.34162131945292157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,7,0.2955840031305949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,15,0.4188106854756673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,15,0.3701333204905192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,31,0.41927464803059894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,31,0.37166933218638104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,63,0.4213493267695109
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,63,0.3737599849700928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,127,0.4971199830373128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,127,0.45425065358479816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,float16,255,0.756981372833252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,1,0.3264053265253703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,3,0.3368426561355591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,1,128,0,1,float16,fp8,255,0.7076853116353353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,1,0.2698240081469218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,3,0.28191999594370526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,7,0.2960853377978007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,7,0.3441386620203654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,15,0.3702293237050374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,15,0.4196853240331014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,31,0.3717386722564697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,31,0.4183093309402466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,63,0.421125332514445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,63,0.37445334593455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,127,0.45401068528493244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,127,0.4984106620152791
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,float16,255,0.7566933631896973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,1,0.32359999418258667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,1,0.27129065990448
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,2,128,0,1,float16,fp8,255,0.7098399798075358
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,3,0.33710400263468426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,3,0.28148800134658813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,7,0.3447200059890747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,7,0.29604266087214154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,15,0.3705973227818807
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,15,0.4189866781234741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,31,0.4190080165863037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,31,0.3720906575520833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,63,0.3739413420359294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,63,0.42101867993672687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,127,0.49861331780751544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,127,0.4557120005289714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,1,0.037258667250474296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,1,0.036090667049090065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,fp8,255,0.7097333272298177
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,3,0.037274666130542755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,4,128,0,1,float16,float16,255,0.7820906639099121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,3,0.03597866743803024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,7,0.036703998843828835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,7,0.035317334036032356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,15,0.03711466739575068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,15,0.03602666656176249
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,31,0.03573333223660787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,31,0.03722133239110311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,63,0.03783999880154928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,63,0.03611200054486593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,127,0.03789866715669632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,127,0.0359946663180987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,float16,255,0.04186666508515676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,128,8,128,0,1,float16,fp8,255,0.03741333385308584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,1,0.5313813289006551
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,1,0.6423733234405518
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,3,0.668351968129476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,3,0.5544799963633219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,7,0.5825440088907877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,7,0.6766080061594645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,15,0.7338613669077555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,15,0.8309866587320963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,31,0.7362826665242513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,31,0.8302773634592692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,63,0.739408016204834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,63,0.8349173069000244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,fp8,127,0.9002026716868082
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,1,128,0,1,float16,float16,127,0.9875840346018473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,1,0.5310080051422119
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,1,0.6419466733932495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,3,0.5555626551310221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,3,0.6684479713439941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,7,0.6829333305358887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,7,0.5846133232116699
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,15,0.7338986396789551
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,15,0.8303573131561279
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,31,0.8309546311696371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,31,0.7366186777750651
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,63,0.8353599707285563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,63,0.739957332611084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,float16,127,0.9904159704844157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,2,128,0,1,float16,fp8,127,0.9003840287526449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,1,0.6396960020065308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,1,0.5353920062383016
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,3,0.6686453024546305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,3,0.5559626817703247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,7,0.683626651763916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,7,0.5842666625976562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,15,0.831109364827474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,15,0.7354613145192465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,31,0.7366186777750651
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,31,0.8319306373596191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,63,0.8388693332672119
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,63,0.7415946324666342
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,fp8,127,0.9038826624552408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,4,128,0,1,float16,float16,127,1.0305866400400798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,1,0.06464533507823944
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,3,0.06452799836794536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,3,0.06039999922116598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,1,0.06039466460545858
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,7,0.06465066472689311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,7,0.06054399907588959
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,15,0.06452266871929169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,15,0.060346667965253196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,31,0.06404266754786174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,31,0.06005333364009857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,63,0.06459733347098033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,float16,127,0.06516799827416737
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,1,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,63,0.06035199761390686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,1,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,3,0.018922666708628338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,3,0.015360000232855478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,7,0.018976000448067982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,7,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,128,8,128,0,1,float16,fp8,127,0.06247466802597046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,15,0.019551999866962433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,15,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,31,0.019567999988794327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,31,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,63,0.019546666493018467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,63,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,127,0.023221333821614582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,127,0.021114667256673176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,255,0.0317546675602595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,255,0.02934933453798294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,511,0.049642667174339294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,511,0.04764266808827718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,1023,0.08467732866605122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,1023,0.08323200047016144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,2047,0.15471999843915304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,2047,0.15255467096964517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,4095,0.29630400737126666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,4095,0.29208000500996906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,8191,0.5805759827295939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,8191,0.5721439917882284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,1,0.01865600049495697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,1,0.01471466695268949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,3,0.018842666099468868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,3,0.015413332730531693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,7,0.018309333672126133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,7,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,15,0.019306667149066925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,15,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,31,0.019178666174411774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,31,0.01844800015290578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,float16,16383,1.215183973312378
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,63,0.01929066702723503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,63,0.018842666099468868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,1,128,0,1,float16,fp8,16383,1.3694079717000325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,127,0.022858666876951855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,127,0.021045332153638203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,255,0.03141866624355316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,255,0.029722665747006733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,511,0.047781333327293396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,511,0.048058668772379555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,1023,0.08482133348782857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,1023,0.08273600041866302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,2047,0.15448533495267233
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,2047,0.15249066551526388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,4095,0.2986133297284444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,4095,0.2916640043258667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,8191,0.5838559865951538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,8191,0.5720266501108805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,1,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,1,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,3,0.018101333330074947
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,3,0.015370666980743408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,7,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,7,0.015344000111023584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,15,0.019061333189407986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,15,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,31,0.019013332823912304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,float16,16383,1.3012266953786213
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,2,128,0,1,float16,fp8,16383,1.3681227366129558
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,31,0.01720000058412552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,63,0.019359999646743137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,63,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,127,0.023210667073726654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,127,0.021701333423455555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,255,0.03155199935038885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,255,0.029792000850041706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,511,0.049813335140546165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,511,0.04798933366934458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,1023,0.0846506655216217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,1023,0.08292800188064575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,2047,0.15465600291887918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,2047,0.15255467096964517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,4095,0.29786133766174316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,4095,0.2921813329060872
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,8191,0.5880373318990072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,8191,0.573855996131897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,1,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,3,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,7,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,float16,16383,1.4043199221293132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,7,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,15,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,15,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,31,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,4,128,0,1,float16,fp8,16383,1.4121972719828289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,31,0.012698666503032049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,63,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,63,0.011952000359694162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,127,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,127,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,255,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,255,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,511,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,511,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,1023,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,1023,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,2047,0.019487999379634857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,2047,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,4095,0.023002666731675465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,4095,0.021317332983016968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,8191,0.038831998904546104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,8191,0.027349332968393963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,float16,16383,0.06031466523806254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,128,8,128,0,1,float16,fp8,16383,0.042447999119758606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,1,1.278282642364502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,1,1.0541653633117676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,3,1.3284320036570232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,3,1.1030453046162922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,7,1.1618133385976155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,7,1.3511199951171875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,15,1.6581813494364421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,15,1.4616959889729817
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,31,1.4681866963704426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,31,1.6589600245157878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,float16,63,1.683882713317871
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,1,128,0,1,float16,fp8,63,1.4774346351623535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,1,1.0581066608428955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,1,1.2733653386433919
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,3,1.3328959941864014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,3,1.1067732969919841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,7,1.1629172960917156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,7,1.3654346466064453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,15,1.6635146141052246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,15,1.4639573097229004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,31,1.4746987024943035
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,31,1.675264040629069
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,float16,63,1.7138773600260417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,1,1.06441068649292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,1,1.2739840348561604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,2,128,0,1,float16,fp8,63,1.4951839447021484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,3,1.1089173158009846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,3,1.3438080151875813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,7,1.178650697072347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,7,1.3765974044799805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,15,1.4746826489766438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,15,1.6801652908325195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,31,1.4952534039815266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,31,1.7048479715983074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,1,0.11583466331164043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,1,0.11145599683125813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,3,0.11544000109036763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,fp8,63,1.5353546142578125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,3,0.11170666416486104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,4,128,0,1,float16,float16,63,1.7245546976725261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,7,0.11589866876602173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,7,0.1111253301302592
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,15,0.11658133069674174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,15,0.11142399907112122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,31,0.1165120005607605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,31,0.11157332857449849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,fp8,63,0.11157866319020589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,128,8,128,0,1,float16,float16,63,0.11684800187746684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,1,2.2703893979390464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,1,2.7182238896687827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,3,2.8076000213623047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,3,2.3184693654378257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,7,2.412725289662679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,7,2.8466612497965493
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,15,2.9924373626708984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,15,3.3875999450683594
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,fp8,31,2.996357282002767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,1,128,0,1,float16,float16,31,3.3835840225219727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,1,2.3022133509318032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,1,2.752079963684082
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,3,2.337301254272461
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,3,2.8306665420532227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,7,2.4390719731648765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,7,2.87992000579834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,15,3.0130081176757812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,15,3.3997812271118164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,float16,31,3.3980000813802085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,2,128,0,1,float16,fp8,31,3.017024040222168
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,1,2.3389652570088706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,1,2.809856096903483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,3,2.381381352742513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,3,2.86846923828125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,7,2.4695040384928384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,7,2.9155041376749673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,15,3.0209760665893555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,15,3.4151201248168945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,fp8,31,3.033365249633789
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,4,128,0,1,float16,float16,31,3.4158827463785806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,1,0.2193386753400167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,1,0.2097439964612325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,3,0.2207253376642863
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,3,0.21058666706085205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,7,0.22034666935602823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,7,0.20958934227625528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,15,0.22049599885940552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,15,0.21030932664871216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,1,0.02736533433198929
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,1,0.023408000667889912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,3,0.027600000301996868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,3,0.025621332228183746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,7,0.027744000156720478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,7,0.025029333929220837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,15,0.03234133372704188
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,15,0.029631999631722767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,31,0.03183466692765554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,31,0.029951999584833782
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,63,0.03259200106064478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,63,0.030085332691669464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,fp8,31,0.21067200104395548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,127,0.03801066676775614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,127,0.035536001125971474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,255,0.05570133527119955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,255,0.05241066714127859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,511,0.08928533395131429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,511,0.08744532863299052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,1023,0.15865600109100342
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,1023,0.15520532925923666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,2047,0.2972106734911601
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,2047,0.28918933868408203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,4095,0.5748639901479086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,4095,0.5577280124028524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,1,0.027562665442625683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,1,0.02403733382622401
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,float16,8191,1.1287946701049805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,3,0.027477333943049114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,3,0.025242666403452556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,7,0.02531733363866806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,7,0.027722666660944622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,1,128,0,1,float16,fp8,8191,1.0991520086924236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,15,0.03311466674009959
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,15,0.02991466720898946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,31,0.029450667401154835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,31,0.033861334125200905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,63,0.0334346666932106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,63,0.03030933439731598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,127,0.03788800040880839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,255,0.05612266560395559
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,127,0.03583466758330663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,255,0.05279466509819031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,511,0.09083200494448344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,511,0.08726400136947632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,1023,0.15913599729537964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,1023,0.15461867054303488
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,2047,0.29792000850041706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,2047,0.28990934292475384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,4095,0.5751946767171224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,4095,0.55840531984965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,1,0.027386667827765148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,1,0.023573334018389385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,3,0.02734400083621343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,float16,8191,1.1497173309326172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,3,0.025594666600227356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,2,128,0,1,float16,fp8,8191,1.0981067021687825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,7,0.02757866680622101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,7,0.02537599951028824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,15,0.03254933406909307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,15,0.029818666477998097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,63,0.03347733368476232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,31,0.02935466667016347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,31,0.032032000521818794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,128,8,128,0,1,float16,float16,31,0.221343994140625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,63,0.02958933264017105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,127,0.03762666632731756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,255,0.05596266686916351
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,255,0.052149335543314614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,127,0.035418666899204254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,511,0.08701333403587341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,511,0.09093333284060161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,1023,0.15896000464757284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,1023,0.15431466698646545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,2047,0.2975200017293294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,2047,0.29047467311223346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,4095,0.5849013328552246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,4095,0.5576266845067342
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,1,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,1,0.011722666521867117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,3,0.011781333635250727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,float16,8191,1.1593866348266602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,7,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,7,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,15,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,4,128,0,1,float16,fp8,8191,1.115290641784668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,15,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,31,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,31,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,63,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,63,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,127,0.012650666137536367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,127,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,255,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,255,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,511,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,511,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,1023,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,1023,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,2047,0.0215786670645078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,2047,0.021194666624069214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,4095,0.027093333502610523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,4095,0.03757333258787791
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,1,0.03755733370780945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,float16,8191,0.059749335050582886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,128,8,128,0,1,float16,fp8,8191,0.041936000188191734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,1,0.03745066622893015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,1,0.03198933353026708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,1,0.03347733368476232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,3,0.03741333385308584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,3,0.03741333385308584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,3,0.03329599897066752
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,3,0.03312533348798752
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,7,0.03813866774241129
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,7,0.039135999977588654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,7,0.033370666205883026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,7,0.03330666571855545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,15,0.039621333281199135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,15,0.039808000127474465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,15,0.033439998825391136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,15,0.03391999999682108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,31,0.04601066807905833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,31,0.04580266773700714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,31,0.041434665520985924
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,31,0.04173333446184794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,63,0.046053335070610046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,63,0.04568533102671305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,63,0.0417546679576238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,63,0.0418453315893809
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,127,0.04764799773693085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,127,0.04790933430194855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,127,0.041850666205088295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,255,0.04779199759165446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,255,0.055061335364977516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,127,0.042538667718569435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,255,0.041840001940727234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,255,0.050016000866889954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,511,0.048063998421033226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,511,0.08101333181063335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,511,0.042709335684776306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,511,0.0763679991165797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,1023,0.047728002071380615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,1023,0.13210666179656982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,1023,0.04233066737651825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,1023,0.12651733557383218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,2047,0.047797332207361855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,2047,0.04233600199222565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,float16,4095,0.04774933556715647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,2047,0.2347093423207601
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,2047,0.2301973303159078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,1,0.03775466730197271
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,128,1,float16,fp8,4095,0.04186133543650309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,1,0.03771200031042099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,1,0.0313226655125618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,fp8,4095,0.43530134359995526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,1,0.03331200033426285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,64,0,1,float16,float16,4095,0.4427573283513387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,3,0.03775466730197271
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,3,0.033376000821590424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,3,0.0377866675456365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,7,0.03756266583998998
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,3,0.03340800106525421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,7,0.03975466638803482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,7,0.03387200087308884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,7,0.03400533397992452
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,15,0.03973866750796636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,15,0.03968533376852671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,15,0.03433600068092346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,15,0.033999999364217125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,31,0.046709333856900535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,31,0.046341334780057274
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,31,0.04225599765777588
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,31,0.042303999265034996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,63,0.04719999929269155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,63,0.04651199777921041
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,63,0.042117332418759666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,63,0.042208001017570496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,127,0.04828266799449921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,127,0.048197334011395775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,127,0.04236799975236257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,127,0.04250133534272512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,255,0.05598933498064677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,255,0.04795733094215393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,255,0.04195733368396759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,255,0.050053333242734276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,511,0.04811733464399973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,511,0.08084266881148021
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,511,0.0418453315893809
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,511,0.07728533446788788
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,1023,0.04775999983151754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,1023,0.13212266564369202
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,1023,0.042250668009122215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,1023,0.12658666570981345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,2047,0.04794666667779287
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,2047,0.04193066557248434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,2047,0.23492266734441122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,2047,0.2305813431739807
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,float16,4095,0.04764266808827718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,128,1,float16,fp8,4095,0.04189866781234741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,float16,4095,0.4480959971745809
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,1,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,1,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,1,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,64,0,1,float16,fp8,4095,0.4358186721801758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,1,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,3,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,3,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,3,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,7,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,3,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,7,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,7,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,7,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,15,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,15,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,15,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,15,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,31,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,31,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,31,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,63,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,31,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,63,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,63,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,63,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,127,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,127,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,127,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,127,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,255,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,255,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,255,0.013999999811251959
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,255,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,511,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,511,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,511,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,1023,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,511,0.013717333475748697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,1023,0.01732800031701724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,1023,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,1023,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,2047,0.01618133361140887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,2047,0.025589334468046825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,2047,0.015344000111023584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,2047,0.025424001117547352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,float16,4095,0.04213866591453552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,float16,4095,0.01613333324591319
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,128,1,float16,fp8,4095,0.015552000453074774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,64,0,1,float16,fp8,4095,0.03563733398914337
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,1,0.009162666896979014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,1,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,1,0.009338666374484697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,3,0.009242666885256767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,3,0.009039999917149544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,3,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,7,0.009098666409651438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,7,0.009450666606426239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,7,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,7,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,15,0.009141333401203156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,15,0.008992000172535578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,15,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,15,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,31,0.009205333267649015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,31,0.009232000137368837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,31,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,31,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,63,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,63,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,63,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,127,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,127,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,127,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,127,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,255,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,255,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,255,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,255,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,511,0.019248000035683315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,511,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,511,0.021402666966120403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,511,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,1023,0.019567999988794327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,1023,0.02348800003528595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,1023,0.02363733450571696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,1023,0.02754666656255722
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,2047,0.019754666835069656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,2047,0.029466666281223297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,2047,0.025045332809289295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,2047,0.03198933353026708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,4095,0.01937599976857503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,4095,0.04199466605981191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,4095,0.02369600037733714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,4095,0.042362665136655174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,8191,0.021007999777793884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,8191,0.06496533254782359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,8191,0.0621013343334198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,8191,0.023711999257405598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,float16,16383,0.02013333390156428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,128,1,float16,fp8,16383,0.023919999599456787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,float16,16383,0.11116799712181091
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,1,0.009589333087205887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,1,0.009301333377758661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,1,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,64,0,1,float16,fp8,16383,0.10121599833170573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,3,0.009114666531483332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,3,0.009306666751702627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,3,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,3,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,7,0.00903466654320558
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,7,0.00922133338948091
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,15,0.009375999992092451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,15,0.009066666786869368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,31,0.010490667074918747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,15,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,31,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,31,0.009402666861812273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,63,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,63,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,63,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,127,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,127,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,127,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,127,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,255,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,255,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,255,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,255,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,511,0.019573333362738293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,511,0.021333334346612293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,511,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,511,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,1023,0.019626667102177937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,1023,0.02380266785621643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,1023,0.023189333577950794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,1023,0.0276053324341774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,2047,0.01934933289885521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,2047,0.029626667499542236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,2047,0.024143998821576435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,2047,0.03166933357715607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,4095,0.019194666296243668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,4095,0.041893333196640015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,4095,0.023141334454218548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,4095,0.041573333243529
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,8191,0.01978133370478948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,8191,0.02363733450571696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,8191,0.06485866506894429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,8191,0.06364266574382782
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,float16,16383,0.019424000134070713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,float16,16383,0.10974933703740437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,128,1,float16,fp8,16383,0.023344000180562336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,1,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,1,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,1,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,64,0,1,float16,fp8,16383,0.10105066498120625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,3,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,3,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,7,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,7,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,15,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,15,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,15,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,31,0.01184533288081487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,31,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,31,0.01201066623131434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,63,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,63,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,63,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,63,0.011866666376590729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,127,0.011690666278203329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,127,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,127,0.011930666863918304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,127,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,255,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,255,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,255,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,255,0.012159999459981918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,511,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,511,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,511,0.012106666962305704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,511,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,1023,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,1023,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,1023,0.012261333564917246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,1023,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,2047,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,2047,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,2047,0.012639999389648438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,2047,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,4095,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,4095,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,4095,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,4095,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,8191,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,8191,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,8191,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,8191,0.01580799991885821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,float16,16383,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,float16,16383,0.0174346665541331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,1,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,128,1,float16,fp8,16383,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,1,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,64,0,1,float16,fp8,16383,0.017674667139848072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,1,0.016821333517630894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,1,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,3,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,3,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,3,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,3,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,7,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,7,0.013701333353916803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,7,0.018485333770513535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,7,0.017184000462293625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,15,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,15,0.014896000425020853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,15,0.016879999389251072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,15,0.01579733317097028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,31,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,31,0.014741333822409311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,31,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,31,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,63,0.015114666273196539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,63,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,63,0.017765333255132038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,63,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,127,0.015466666469971338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,127,0.015893333901961643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,127,0.021541332205136616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,127,0.021136000752449036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,255,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,255,0.015685333559910457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,255,0.021333334346612293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,255,0.02145066608985265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,511,0.015599999576807022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,511,0.02145066608985265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,511,0.017562666287024815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,511,0.02141333371400833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,1023,0.016895999511082966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,1023,0.023183998962243397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,1023,0.02130666623512904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,1023,0.025349333882331848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,2047,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,2047,0.033402666449546814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,2047,0.021082667013009388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,2047,0.03425599883000056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,4095,0.016917333006858826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,4095,0.021536000072956085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,4095,0.05531733234723409
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,4095,0.05434666574001312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,8191,0.016832000265518825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,8191,0.021498667697111767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,8191,0.09674666325251262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,8191,0.09153067072232564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,float16,16383,0.01759999990463257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,128,1,float16,fp8,16383,0.02125866711139679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,float16,16383,0.17814399798711142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,1,0.014778666198253632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,1,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,64,0,1,float16,fp8,16383,0.16735466321309408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,1,0.016986666868130367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,1,0.01775466650724411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,3,0.014549333602190018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,3,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,3,0.01687466725707054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,3,0.017498667041460674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,7,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,7,0.014752000570297241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,7,0.017194667210181553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,7,0.017770666629076004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,15,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,15,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,15,0.017717332889636356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,15,0.01747200017174085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,31,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,31,0.01563199982047081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,31,0.016943999876578648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,31,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,63,0.014805333067973455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,63,0.015647999942302704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,63,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,63,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,127,0.016688000410795212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,127,0.015562667200962702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,127,0.02139200021823247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,127,0.021738665799299877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,255,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,255,0.015696000307798386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,255,0.02149333308140437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,255,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,511,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,511,0.021269333859284718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,511,0.017514667163292568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,511,0.021333334346612293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,1023,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,1023,0.021349333226680756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,1023,0.023402666052182514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,1023,0.025050667424996693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,2047,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,2047,0.03376533339420954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,2047,0.02146666745344798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,2047,0.03486400097608566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,4095,0.01609066625436147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,4095,0.054058666030565895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,4095,0.02164799968401591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,4095,0.05449600021044413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,8191,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,8191,0.09553066889444987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,8191,0.02181333303451538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,8191,0.09108266234397888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,float16,16383,0.015573333948850632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,128,1,float16,fp8,16383,0.021397332350413006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,float16,16383,0.17891732851664224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,1,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,64,0,1,float16,fp8,16383,0.16710400581359863
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,3,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,1,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,3,0.01209066684047381
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,3,0.011610666910807291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,7,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,7,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,7,0.011653333902359009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,15,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,7,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,15,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,15,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,31,0.011866666376590729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,31,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,31,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,31,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,63,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,63,0.012202666451533636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,63,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,127,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,127,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,127,0.012213333199421564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,255,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,255,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,255,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,511,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,255,0.011989332735538483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,511,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,511,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,511,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,1023,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,1023,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,1023,0.01258133351802826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,1023,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,2047,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,2047,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,2047,0.013845333208640417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,2047,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,4095,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,4095,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,4095,0.01533866673707962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,4095,0.015856000284353893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,8191,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,8191,0.01800000046690305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,8191,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,8191,0.017637333522240322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,float16,16383,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,float16,16383,0.020975999534130096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,128,1,float16,fp8,16383,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,1,0.06866133213043213
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,64,0,1,float16,fp8,16383,0.019786667078733444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,1,0.06885333359241486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,1,0.056287998954455055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,1,0.05632533133029938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,3,0.06855466465155284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,3,0.06845333178838094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,3,0.05681066711743673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,3,0.056474665800730385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,7,0.07042133311430614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,7,0.0709386666615804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,7,0.05832533538341522
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,7,0.05820799867312113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,15,0.0711839993794759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,15,0.07213866710662842
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,15,0.061573331554730736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,15,0.060229331254959106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,31,0.08498133222262065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,31,0.08499200145403545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,31,0.07509866853555043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,31,0.07620266576608022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,63,0.0862506628036499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,63,0.0862506628036499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,63,0.07684266567230225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,63,0.07643199960390727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,127,0.08672533432642619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,127,0.08710400263468425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,127,0.07644799848397572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,127,0.07678933441638947
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,255,0.08702400326728821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,255,0.1016533374786377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,255,0.07659733295440674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,255,0.09118933478991191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,511,0.08703466256459554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,511,0.15034133195877075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,511,0.07635200023651123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,511,0.14202666282653809
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,1023,0.08692800005276997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,1023,0.07666666805744171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,1023,0.24865599473317465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,1023,0.23860265811284384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,float16,2047,0.08709333340326945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,128,1,float16,fp8,2047,0.07643199960390727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,float16,2047,0.44973333676656085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,1,0.06825600067774455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,1,0.05608533322811127
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,64,0,1,float16,fp8,2047,0.43885334332784015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,1,0.069541335105896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,1,0.05588266750176748
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,3,0.06835199892520905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,3,0.06840000053246816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,3,0.056373332937558494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,3,0.057674666245778404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,7,0.07045333087444305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,7,0.070592001080513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,7,0.05857066810131073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,15,0.07245866457621257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,15,0.07262933254241943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,7,0.059205333391825356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,15,0.06002133091290792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,15,0.0602453351020813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,31,0.08493866523106892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,31,0.08557867010434468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,31,0.07482666770617168
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,31,0.07641066610813141
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,63,0.08646933237711589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,63,0.07630933324495952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,63,0.07643199960390727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,63,0.08737599849700928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,127,0.08711999654769897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,127,0.08666132887204488
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,127,0.07666666805744171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,255,0.08682666222254436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,127,0.0769760012626648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,255,0.10121066371599834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,255,0.07683733105659485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,255,0.09257066249847412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,511,0.08719467123349507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,511,0.150709331035614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,511,0.07692799965540568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,511,0.14287466804186502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,1023,0.08713600039482117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,1023,0.24904000759124756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,1023,0.07699733475844066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,1023,0.24086399873097739
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,float16,2047,0.08724266290664673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,128,1,float16,fp8,2047,0.07727999985218048
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,float16,2047,0.4559359947840373
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,1,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,1,0.015530666957298914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,64,0,1,float16,fp8,2047,0.4397226572036743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,1,0.015386667102575302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,1,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,3,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,3,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,3,0.015471999843915304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,3,0.015509333461523056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,7,0.015413332730531693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,7,0.01532799998919169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,7,0.015743999431530636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,7,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,15,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,15,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,15,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,15,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,31,0.01540800059835116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,31,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,31,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,31,0.015568000574906668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,63,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,63,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,63,0.015669333438078564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,63,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,127,0.015450666348139444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,127,0.016522667060295742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,127,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,127,0.01569066693385442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,255,0.016517333686351776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,255,0.015626666446526844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,255,0.016565332810084026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,255,0.01573333392540614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,511,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,511,0.018874666343132656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,511,0.01533866673707962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,511,0.01959466685851415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,1023,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,1023,0.025610665480295818
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,1023,0.017184000462293625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,1023,0.02346133440732956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,float16,2047,0.017466666797796886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,float16,2047,0.044064000248909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,128,1,float16,fp8,2047,0.018906666586796444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,1,0.01911466692884763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,1,0.01897066707412402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,64,0,1,float16,fp8,2047,0.03774400055408478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,1,0.015802666544914246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,1,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,3,0.019567999988794327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,3,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,3,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,3,0.015450666348139444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,7,0.019445333629846573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,7,0.018885333091020584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,7,0.01578666642308235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,7,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,15,0.01916266605257988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,15,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,15,0.020314666132132213
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,15,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,31,0.019391999890406925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,31,0.01970133309563001
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,31,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,31,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,63,0.021381333470344543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,63,0.021151999632517498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,63,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,63,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,127,0.02124800036350886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,127,0.021386665602525074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,127,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,127,0.016938666502634685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,255,0.021514666577180225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,255,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,255,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,255,0.017071999609470367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,511,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,511,0.02359466751416524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,511,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,511,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,1023,0.021333334346612293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,1023,0.03166933357715607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,1023,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,1023,0.02720000098148982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,2047,0.021568000316619873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,2047,0.04781333108743032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,2047,0.01720000058412552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,2047,0.041375999649365745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,4095,0.021557333568731945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,4095,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,4095,0.07852266728878021
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,4095,0.07121600210666656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,8191,0.021685334543387096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,8191,0.14054399728775024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,8191,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,8191,0.13010133306185404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,float16,16383,0.021354667842388153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,float16,16383,0.26524267594019574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,128,1,float16,fp8,16383,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,1,0.019280000279347103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,1,0.01894933357834816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,64,0,1,float16,fp8,16383,0.24898133675257364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,1,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,1,0.015471999843915304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,3,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,3,0.019274666905403137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,3,0.015504000087579092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,3,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,7,0.019253333409627277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,7,0.01911466692884763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,7,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,7,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,15,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,15,0.018800000349680584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,15,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,15,0.015647999942302704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,31,0.01923199991385142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,31,0.02107200026512146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,31,0.016650666793187458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,31,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,63,0.021226666867733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,63,0.02189333240191142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,63,0.01699200024207433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,63,0.017845333864291508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,127,0.021541332205136616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,127,0.0216799999276797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,127,0.017727999637524288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,255,0.02143466720978419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,127,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,255,0.021381333470344543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,255,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,255,0.017360000560681026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,511,0.021344001094500225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,511,0.024277334411938984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,511,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,511,0.01942933350801468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,1023,0.02130666623512904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,1023,0.033439998825391136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,1023,0.02741333345572154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,1023,0.01718933383623759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,2047,0.02162666618824005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,2047,0.04837333162625631
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,2047,0.017290666699409485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,2047,0.041797334949175514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,4095,0.021205333371957142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,4095,0.07879466811815898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,4095,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,4095,0.0724480003118515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,8191,0.021045332153638203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,8191,0.14122133453687033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,8191,0.01752000053723653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,8191,0.1305333375930786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,float16,16383,0.021562665700912476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,float16,16383,0.26499734322230023
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,128,1,float16,fp8,16383,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,1,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,1,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,64,0,1,float16,fp8,16383,0.24861866235733032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,1,0.011898666620254517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,1,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,3,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,3,0.011616000284751257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,7,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,7,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,7,0.01231466606259346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,7,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,15,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,15,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,15,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,31,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,31,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,31,0.012181332955757776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,63,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,63,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,63,0.012719999998807907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,63,0.012469333906968435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,127,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,127,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,127,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,127,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,255,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,255,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,255,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,511,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,255,0.011829332758982977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,511,0.012245333443085352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,511,0.012234666695197424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,1023,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,511,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,1023,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,1023,0.012682666381200155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,1023,0.013631999492645264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,2047,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,2047,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,2047,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,2047,0.014122666170199713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,4095,0.01351999988158544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,4095,0.01714666684468587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,4095,0.013936000565687815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,4095,0.017045332739750545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,8191,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,8191,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,8191,0.020997333029905956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,8191,0.01926400015751521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,float16,16383,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,float16,16383,0.026389333109060924
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,128,1,float16,fp8,16383,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,64,0,1,float16,fp8,16383,0.023306667804718018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,1,0.1305333375930786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,1,0.10538666447003682
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,1,0.12821333607037863
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,3,0.13013866543769836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,1,0.10598400235176086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,3,0.12826133767763773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,3,0.10517866412798564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,3,0.10523200035095215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,7,0.13326399525006613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,7,0.13382400075594583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,7,0.10958400368690491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,7,0.10954667131106059
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,15,0.13617600003878275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,15,0.11359999577204387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,15,0.13607466220855713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,15,0.11366933584213257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,31,0.162800004084905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,31,0.16289066274960837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,31,0.14340800046920776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,31,0.1442026694615682
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,63,0.16503467162450156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,63,0.1646933356920878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,63,0.14422399799029031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,63,0.1446453332901001
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,127,0.166810671488444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,127,0.16743467251459757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,127,0.14646933476130167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,127,0.14638400077819824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,255,0.1666933298110962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,255,0.19579199949900308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,255,0.14643733700116476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,255,0.17495467265446982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,511,0.1672053337097168
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,511,0.2930613358815511
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,float16,1023,0.16744534174601236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,511,0.273087998231252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,511,0.14696533481280008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,float16,1023,0.4859040180842082
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,128,1,float16,fp8,1023,0.14615999658902487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,1,0.12988266348838806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,64,0,1,float16,fp8,1023,0.46596264839172363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,1,0.10537599523862202
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,1,0.12965866923332214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,1,0.10545066992441814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,3,0.1280693312486013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,3,0.1299253304799398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,3,0.1058079997698466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,3,0.10587732990582784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,7,0.13190933068593344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,7,0.13366400202115378
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,7,0.10988266269365947
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,7,0.1095360020796458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,15,0.13613866766293845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,15,0.13621866703033447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,15,0.11358933647473653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,15,0.11395733555157979
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,31,0.16270933548609415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,31,0.1632266640663147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,31,0.14415466785430908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,31,0.14402666687965393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,63,0.16470932960510254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,63,0.16487999757130942
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,63,0.14447999993960062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,63,0.14494933684666952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,127,0.1668000022570292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,127,0.16671999295552573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,127,0.14623467127482095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,127,0.14591466387112936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,255,0.16685332854588827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,255,0.19548799594243368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,255,0.1460693379243215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,255,0.17517866690953574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,511,0.16684265931447348
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,511,0.14607466260592142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,511,0.29224000374476117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,511,0.2736799915631612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,float16,1023,0.1666826605796814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,128,1,float16,fp8,1023,0.14587733149528503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,float16,1023,0.49848000208536786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,1,0.01918399954835574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,1,0.019306667149066925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,1,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,64,0,1,float16,fp8,1023,0.46640535195668537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,1,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,3,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,3,0.019178666174411774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,3,0.021205333371957142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,3,0.01918399954835574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,7,0.019120000302791595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,7,0.01924266666173935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,7,0.019253333409627277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,7,0.019359999646743137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,15,0.01945066700379054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,15,0.019434666881958645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,15,0.0194560003777345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,15,0.019648000597953796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,31,0.019354666272799175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,31,0.019237333287795384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,31,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,31,0.021168000996112823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,63,0.019343999524911244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,63,0.019610666980346043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,63,0.021045332153638203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,63,0.022624000906944275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,127,0.01952533299724261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,127,0.01942933350801468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,127,0.01940800001223882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,127,0.020997333029905956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,255,0.019941333681344986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,255,0.019130667050679524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,255,0.021146667500336964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,255,0.01924266666173935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,511,0.019802667200565338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,511,0.02550933261712392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,511,0.020842666427294414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,511,0.025392000873883564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,float16,1023,0.02128533273935318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,float16,1023,0.03807466725508372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,128,1,float16,fp8,1023,0.020810666183630627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,64,0,1,float16,fp8,1023,0.03528533379236857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,1,0.2509546677271525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,1,0.2508586645126343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,1,0.20171199242273966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,1,0.2016693353652954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,3,0.25064533948898315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,3,0.2510880033175151
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,3,0.20165334145228067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,3,0.20151466131210327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,7,0.2590133349100749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,7,0.25911466280619305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,7,0.20973867177963257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,15,0.2646133303642273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,7,0.21261332432428995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,15,0.265066663424174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,15,0.22000000874201456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,15,0.21968533595403036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,31,0.27940799792607623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,31,0.32018667459487915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,31,0.32103466987609863
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,31,0.27935999631881714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,63,0.32236266136169434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,63,0.3242560029029846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,63,0.28220800558725995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,63,0.2818079988161723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,127,0.3261013428370158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,127,0.3246506651242574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,127,0.2842293381690979
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,127,0.2833706736564636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,float16,255,0.326581339041392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,float16,255,0.3835466702779134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,128,1,float16,fp8,255,0.2839573423067729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,64,0,1,float16,fp8,255,0.3429493506749471
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,1,0.2510293324788411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,1,0.2507999936739604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,1,0.20440000295639038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,1,0.2036959926287333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,3,0.2507999936739604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,3,0.2505813241004944
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,3,0.20393067598342896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,3,0.2041119933128357
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,7,0.2590453426043193
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,7,0.25882667303085327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,7,0.21202667554219565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,7,0.21220799287160239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,15,0.26477867364883423
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,15,0.26498132944107056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,15,0.22032533089319864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,15,0.22043200333913168
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,31,0.320848007996877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,31,0.32076799869537354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,31,0.27974933385849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,31,0.279968003431956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,63,0.3242186705271403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,63,0.32261866331100464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,63,0.283077339331309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,63,0.28230400880177814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,127,0.3261866569519043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,127,0.32658666372299194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,127,0.2839360038439433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,127,0.2837066650390625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,float16,255,0.3266506592432658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,float16,255,0.38412801424662274
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,1,0.02956266701221466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,128,1,float16,fp8,255,0.2834720015525818
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,64,0,1,float16,fp8,255,0.3429866631825765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,1,0.029178666571776073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,1,0.029301332930723827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,1,0.029663999875386555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,3,0.029365333418051403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,3,0.029418667157491047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,3,0.02957333376010259
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,3,0.02956266701221466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,7,0.029466666281223297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,7,0.029343999922275543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,7,0.02980799973011017
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,7,0.02957333376010259
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,15,0.02934933453798294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,15,0.02905600021282832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,15,0.029290666182835896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,15,0.029370665550231934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,31,0.029690665503342945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,31,0.029333333174387615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,31,0.029861333469549816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,31,0.029333333174387615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,63,0.02942399928967158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,63,0.029296000798543293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,63,0.029418667157491047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,63,0.029232000311215717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,127,0.02957333376010259
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,127,0.02932800104220708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,127,0.029290666182835896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,127,0.03027733415365219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,float16,255,0.03142400085926056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,float16,255,0.029578665892283123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,128,1,float16,fp8,255,0.03160533308982849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,64,0,1,float16,fp8,255,0.029482667644818623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,1,0.49294400215148926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,1,0.49482134977976483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,1,0.39581867059071857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,3,0.4941706657409668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,3,0.39549867312113446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,3,0.4948586622873942
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,1,0.39667201042175293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,3,0.39560532569885254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,7,0.5108213424682617
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,7,0.5108853181203207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,7,0.4148799975713094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,7,0.4144853353500366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,15,0.52128533522288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,15,0.5212639967600504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,15,0.43299734592437744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,15,0.4317013422648112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,31,0.631989320119222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,31,0.6337973276774088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,31,0.551797350247701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,31,0.5522880156834921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,63,0.6394613186518351
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,63,0.6398880084355673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,63,0.557808001836141
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,63,0.5565280119578043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,float16,127,0.6440639893213908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,float16,127,0.6443946758906046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,0,1,float16,fp8,127,0.5601813395818075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,64,128,1,float16,fp8,127,0.560917337735494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,1,0.49427199363708496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,1,0.49454398949941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,1,0.3999679883321126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,1,0.40091200669606525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,3,0.4946346680323283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,3,0.40019198258717853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,3,0.4949066638946533
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,3,0.40084266662597656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,7,0.41493332386016846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,7,0.5108640193939209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,7,0.5104586680730184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,7,0.4158453146616618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,15,0.43298665682474774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,15,0.5212213198343912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,15,0.5215040047963461
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,15,0.4336106777191162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,31,0.6319040060043335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,31,0.5522559881210327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,31,0.6332799990971884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,31,0.5524426698684692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,63,0.5577226479848226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,63,0.6397866805394491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,63,0.5575946569442749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,63,0.6402399937311808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,fp8,127,0.560538649559021
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,fp8,127,0.5606880187988281
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,128,1,float16,float16,127,0.6445866823196411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,64,0,1,float16,float16,127,0.6439040104548136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,1,0.04805333415667216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,1,0.04795733094215393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,1,0.04771733283996582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,1,0.047781333327293396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,3,0.04770666857560476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,3,0.047594666481018066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,3,0.04770133395989736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,3,0.04795200129350027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,7,0.048154667019844055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,7,0.04778666794300079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,7,0.04810666541258494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,7,0.04804266492525736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,15,0.04814933240413666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,15,0.04794133206208547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,15,0.047728002071380615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,15,0.04783466458320618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,31,0.047770669062932335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,31,0.04771733283996582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,31,0.04854933420817057
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,31,0.048063998421033226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,63,0.04794666667779287
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,63,0.04782933493455251
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,63,0.047914668917655945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,63,0.0479360024134318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,float16,127,0.048101335763931274
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,float16,127,0.04807466765244802
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,128,1,float16,fp8,127,0.048101335763931274
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,64,0,1,float16,fp8,127,0.0480320006608963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,1,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,1,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,1,0.014127999544143677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,3,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,1,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,3,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,3,0.01482133318980535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,3,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,7,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,7,0.014789332946141561
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,7,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,7,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,15,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,15,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,15,0.014954666296641031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,15,0.01451733335852623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,31,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,31,0.017157333592573803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,31,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,31,0.016997333616018295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,63,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,63,0.017018667111794155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,63,0.017103999853134155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,63,0.016800000021855038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,127,0.016917333006858826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,127,0.01714666684468587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,127,0.017045332739750545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,127,0.016623999923467636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,255,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,255,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,255,0.01907733331123988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,511,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,511,0.027327999472618103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,255,0.01762666677435239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,511,0.016879999389251072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,511,0.026357332865397137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,1023,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,1023,0.01704000060757001
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,1023,0.04171200096607208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,1023,0.04404266675313314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,2047,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,2047,0.07308266560236613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,2047,0.016901332885026932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,4095,0.017386666188637417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,2047,0.07341866691907246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,4095,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,4095,0.1339466671148936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,4095,0.13402133186658224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,8191,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,8191,0.01746133342385292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,8191,0.25736000140508014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,8191,0.25699732700983685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,float16,16383,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,128,1,float16,fp8,16383,0.01695466662446658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,1,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,1,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,1,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,float16,16383,0.5216106573740641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,1,0.014746667196353277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,3,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,3,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,3,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,3,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,64,0,1,float16,fp8,16383,0.5288373231887817
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,7,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,7,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,15,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,7,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,7,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,15,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,15,0.014666666587193808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,15,0.014906667172908783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,31,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,31,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,31,0.01691199963291486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,31,0.015541333705186844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,63,0.017573333034912746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,63,0.01684800038735072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,63,0.017114666601022083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,63,0.017077332983414333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,127,0.017290666699409485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,127,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,127,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,127,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,255,0.017498667041460674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,255,0.020261333634455998
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,255,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,255,0.01904533306757609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,511,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,511,0.027317332724730175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,511,0.016976000120242436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,511,0.027488000690937042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,1023,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,1023,0.04378666480382284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,1023,0.016970666746298473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,1023,0.042261332273483276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,2047,0.017594666530688603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,2047,0.07454933226108551
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,2047,0.01695999999841054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,2047,0.07303466896216075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,4095,0.01749333366751671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,4095,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,4095,0.13648533821105957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,4095,0.13381866614023843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,8191,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,8191,0.017136000096797943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,8191,0.25940267244974774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,8191,0.25643734137217206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,float16,16383,0.017466666797796886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,128,1,float16,fp8,16383,0.016901332885026932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,float16,16383,0.5331626733144125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,1,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,64,0,1,float16,fp8,16383,0.5306880076726278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,1,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,1,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,3,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,3,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,3,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,3,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,7,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,7,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,7,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,7,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,15,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,15,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,15,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,31,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,31,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,31,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,31,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,63,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,63,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,63,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,63,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,127,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,127,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,127,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,255,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,255,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,255,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,511,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,511,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,511,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,511,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,1023,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,1023,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,1023,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,1023,0.013594667116800943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,2047,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,2047,0.01699200024207433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,2047,0.015466666469971338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,2047,0.017583999782800674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,4095,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,4095,0.021205333371957142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,4095,0.01950399950146675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,4095,0.014298666268587112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,8191,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,8191,0.02741866558790207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,8191,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,8191,0.026186667382717133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,float16,16383,0.013418667018413544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,float16,16383,0.04366933306058248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,128,1,float16,fp8,16383,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,64,0,1,float16,fp8,16383,0.035904000202814736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,1,0.9796319802602133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,1,0.7810986836751302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,1,0.9804480075836182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,1,0.7823999722798666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,3,0.9815626939137777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,3,0.7831520239512125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,3,0.9794507026672363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,3,0.7827786604563395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,7,1.0146559874216716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,7,0.8208959897359213
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,7,1.014858643213908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,7,0.8217546939849854
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,15,1.034821351369222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,15,1.035423994064331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,15,0.857754627863566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,15,0.8569920063018799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,31,1.2583626906077068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,31,1.2589866320292156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,31,1.0966133276621501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,31,1.0969493389129639
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,float16,63,1.2733386357625325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,128,1,float16,fp8,63,1.1076213518778484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,float16,63,1.2705600261688232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,64,0,1,float16,fp8,63,1.1074506441752117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,1,0.9815893173217773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,1,0.9806559880574545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,1,0.793450673421224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,1,0.7923786640167236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,3,0.9821759859720866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,3,0.9815253416697184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,3,0.793498675028483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,3,0.7927412986755371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,7,1.015226682027181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,7,0.822282632191976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,7,1.0141493479410808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,7,0.8226079940795898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,15,1.0364000002543132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,15,0.8594079812367758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,15,1.0351306597391765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,15,0.8586826324462891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,31,1.2599146366119385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,31,1.0972053209940593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,31,1.2590826352437336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,31,1.0970453421274822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,float16,63,1.2776106993357341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,128,1,float16,fp8,63,1.1080693403879802
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,float16,63,1.273850679397583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,64,0,1,float16,fp8,63,1.107583999633789
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,1,0.08701333403587341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,1,0.08711999654769897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,1,0.08683733145395915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,1,0.08708266417185466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,3,0.08727999528249104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,3,0.08714133501052856
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,3,0.08747733632723491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,3,0.08702400326728821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,7,0.08711999654769897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,7,0.08678399523099263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,7,0.08699199557304382
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,7,0.08676266670227051
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,15,0.08685866991678874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,15,0.08728532989819844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,15,0.0867733359336853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,15,0.0869653324286143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,31,0.08674666285514832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,31,0.08684800068537395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,31,0.08664533495903015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,31,0.08685333530108134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,float16,63,0.08705600102742513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,128,1,float16,fp8,63,0.08689066767692566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,float16,63,0.08726933598518372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,64,0,1,float16,fp8,63,0.08689600229263306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,1,1.5532372792561848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,1,1.555232048034668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,1,1.9527254104614258
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,1,1.9513546625773113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,3,1.5542292594909668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,3,1.555722713470459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,3,1.950069268544515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,3,1.95195738474528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,7,1.6335840225219727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,7,1.635050614674886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,7,2.018218676249186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,7,2.0182453791300454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,15,1.7060319582621257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,15,1.7059946060180664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,15,2.062373320261637
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,15,2.0673066775004068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,fp8,31,2.1853440602620444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,fp8,31,2.1854027112325034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,128,1,float16,float16,31,2.5090346336364746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,64,0,1,float16,float16,31,2.510298728942871
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,1,1.578229268391927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,1,1.5784746805826824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,1,1.9525599479675293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,1,1.9569493929545085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,3,1.5791254043579102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,3,1.579210599263509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,3,1.9552480379740398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,3,1.9558773040771484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,7,1.6378026008605957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,7,1.6387413342793782
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,7,2.0230612754821777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,7,2.029775937398275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,15,1.7116533915201824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,15,2.0807199478149414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,15,2.0886665980021157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,15,1.713919957478841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,fp8,31,2.193376064300537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,128,1,float16,float16,31,2.5787946383158364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,float16,31,2.5784427324930825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,1,0.16325333714485168
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,1,0.16328533490498862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,1,0.1655893325805664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,64,0,1,float16,fp8,31,2.1938932736714682
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,1,0.1653600037097931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,3,0.1634880006313324
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,3,0.16546666622161865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,3,0.16582933068275452
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,3,0.1634666621685028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,7,0.16331733266512552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,7,0.16312000155448914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,7,0.16542399923006693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,7,0.16538133223851523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,15,0.16295466820398966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,15,0.16536532839139303
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,15,0.16545599699020386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,31,0.1632213294506073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,float16,31,0.1628213326136271
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,1,0.023775999744733173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,1,0.023573334018389385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,fp8,31,0.1649333337942759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,1,0.01953599974513054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,1,0.019306667149066925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,3,0.023530667026837666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,3,0.023669332265853882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,3,0.01932799940307935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,3,0.019861333072185516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,7,0.02359466751416524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,7,0.020432000358899433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,7,0.023520000278949738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,7,0.020362666497627895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,15,0.024160000185171764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,15,0.02362666775782903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,15,0.02102400114138921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,15,0.021338666478792827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,31,0.02754133443037669
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,31,0.028181334336598713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,31,0.02531733363866806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,63,0.027530667682488758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,31,0.026176000634829204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,63,0.027386667827765148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,63,0.025519999365011852
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,63,0.025173333783944447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,127,0.027973333994547527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,127,0.02735466758410136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,127,0.025349333882331848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,127,0.025498665869235992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,255,0.027690666417280834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,255,0.031744000812371574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,255,0.025461333493391674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,511,0.027621333797772724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,255,0.029989334444204967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,511,0.04587199787298838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,511,0.02548266698916753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,511,0.04398933549722036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,1023,0.0281333327293396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,1023,0.07247466842333476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,1023,0.07016533116499583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,1023,0.025744001070658367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,2047,0.027402666707833607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,2047,0.12777599692344666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,2047,0.02566933383544286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,4095,0.02752000093460083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,2047,0.12459199627240498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,4095,0.025631998976071674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,4095,0.2366559902826945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,128,1,float16,float16,15,0.16607999801635742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,float16,8191,0.02754666656255722
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,4095,0.23492266734441122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,128,1,float16,fp8,8191,0.025306666890780132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,1,0.023285334308942158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,float16,8191,0.45902931690216064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,1,0.0235359991590182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,64,0,1,float16,fp8,8191,0.4539253314336141
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,1,0.020021333048741024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,1,0.019237333287795384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,3,0.0234400009115537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,3,0.019274666905403137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,3,0.02401600033044815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,3,0.01934933289885521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,7,0.023413332800070446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,7,0.02386133372783661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,7,0.019205333044131596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,7,0.020784000555674236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,15,0.024432001014550526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,15,0.02350933353106181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,15,0.021183999876181286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,15,0.021349333226680756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,31,0.02741866558790207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,31,0.027402666707833607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,31,0.025253333151340485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,31,0.025216000775496166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,63,0.027488000690937042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,63,0.02743999908367793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,63,0.02518933266401291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,63,0.025407999753952026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,127,0.027301333844661713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,127,0.027647999425729115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,127,0.025434667865435284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,127,0.02553066611289978
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,255,0.02737066646416982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,64,0,1,float16,fp8,31,0.1662773291269938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,255,0.031685332457224526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,255,0.025205334027608235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,255,0.0296426663796107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,511,0.027765333652496338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,511,0.04554133117198944
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,511,0.04388799766699473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,511,0.026127999027570088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,1023,0.029322666426499683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,1023,0.07286400099595387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,1023,0.025813333690166473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,1023,0.07015466690063477
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,2047,0.025173333783944447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,2047,0.027562665442625683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,2047,0.12846933801968893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,2047,0.12528533736864725
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,4095,0.027845333019892376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,4095,0.025311999022960663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,4095,0.2392586668332418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,4095,0.23487999041875204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,float16,8191,0.027834666272004444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,128,1,float16,fp8,8191,0.025424001117547352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,float16,8191,0.45956265926361084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,1,0.011594666788975397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,64,0,1,float16,fp8,8191,0.45370133717854816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,1,0.012138667205969492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,1,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,1,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,3,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,3,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,3,0.012602667013804117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,7,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,7,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,15,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,15,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,15,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,31,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,31,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,31,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,63,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,63,0.012181332955757776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,63,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,63,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,127,0.01184533288081487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,127,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,127,0.013562666873137156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,127,0.011711999773979187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,255,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,255,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,255,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,255,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,511,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,511,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,511,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,511,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,1023,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,1023,0.01228800043463707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,1023,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,1023,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,2047,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,2047,0.019194666296243668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,2047,0.015471999843915304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,2047,0.018778666853904724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,4095,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,4095,0.025786665578683216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,4095,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,4095,0.024154665569464367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,float16,8191,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,128,1,float16,fp8,8191,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,float16,8191,0.04258666435877482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,64,0,1,float16,fp8,8191,0.03367999941110611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,1,0.03796799977620443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,1,0.03368533402681351
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,3,0.03755199909210205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,3,0.03372266640265783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,7,0.03760000069936117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,7,0.03572800010442734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,15,0.04553066690762838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,15,0.04145599901676178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,31,0.04584000011285146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,31,0.041696002086003624
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,63,0.04585599899291992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,63,0.04171733558177948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,127,0.04994133114814758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,127,0.054010664423306785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,255,0.07868266602357228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,255,0.07448000212510426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,511,0.13005866607030234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,511,0.1257973313331604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,1023,0.23428799708684286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,1023,0.2241013248761495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,2047,0.42479467391967773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,2047,0.43957332770029706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,1,0.03613866617282232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,1,0.03331733246644338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,float16,4095,0.8519946734110514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,3,0.038106667498747505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,3,0.033802665770053864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,7,0.037615999579429626
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,1,128,0,1,float16,fp8,4095,0.8208533128102621
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,7,0.03551999976237615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,15,0.046069333950678505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,15,0.04185600082079569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,31,0.045994664231936135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,63,0.04568533102671305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,31,0.041984001795450844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,63,0.04212800165017446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,127,0.053914666175842285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,127,0.049973333875338234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,255,0.07477333148320515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,255,0.07990933458010356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,511,0.1306773324807485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,511,0.12563733259836832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,1023,0.23358400662740073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,1023,0.22451200087865195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,2047,0.44531198342641193
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,2047,0.42422934373219806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,1,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,float16,4095,0.8756053447723389
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,1,0.013760000467300415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,4,128,0,1,float16,fp8,4095,0.8323307037353516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,3,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,3,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,7,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,7,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,15,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,15,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,31,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,31,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,63,0.013408000270525614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,63,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,127,0.01402666668097178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,127,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,255,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,255,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,511,0.0170666662355264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,511,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,1023,0.019189332922299702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,1023,0.019274666905403137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,2047,0.0393653338154157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,2047,0.027610667049884796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,1,0.009125333279371262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,float16,4095,0.06080000102519989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,96,8,128,0,1,float16,fp8,4095,0.04197333256403605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,1,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,3,0.00926399976015091
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,7,0.009370666618148485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,7,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,15,0.009354666496316591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,15,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,31,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,63,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,63,0.013514666507641474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,127,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,127,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,255,0.021525333325068157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,255,0.013408000270525614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,511,0.02402133246262868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,511,0.02720000098148982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,1023,0.029904000461101532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,1023,0.031717332700888314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,2047,0.04141333450873693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,2047,0.04172799984614054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,4095,0.06437866886456807
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,4095,0.06053866446018219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,8191,0.11142399907112122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,8191,0.09917333722114563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,1,0.00898133342464765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,1,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,3,0.00874133345981439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,float16,16383,0.20205867290496826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,3,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,1,128,0,1,float16,fp8,16383,0.17538666725158691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,7,0.009039999917149544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,15,0.009061333412925402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,15,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,31,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,63,0.010608000059922537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,31,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,63,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,127,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,127,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,255,0.021029333273569744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,255,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,511,0.023610666394233704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,511,0.027327999472618103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,1023,0.02958933264017105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,1023,0.031397332747777305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,2047,0.04205866654713949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,2047,0.041696002086003624
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,4095,0.06494399905204773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,4095,0.06052266558011373
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,8191,0.11154133081436157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,8191,0.0997920036315918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,1,0.01071999967098236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,float16,16383,0.20292800664901733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,1,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,4,128,0,1,float16,fp8,16383,0.17537599802017212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,3,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,7,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,15,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,15,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,31,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,63,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,63,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,127,0.011839999506870905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,255,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,255,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,511,0.012736000120639801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,511,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,1023,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,1023,0.012154666086037954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,2047,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,2047,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,4095,0.015487999965747198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,4095,0.016832000265518825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,8191,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,8191,0.017535999417304993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,float16,16383,0.02111999938885371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,1,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,1,0.01613866661985715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,3,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,96,8,128,0,1,float16,fp8,16383,0.01886933296918869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,3,0.01701333373785019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,7,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,7,0.016794666647911072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,15,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,15,0.01701333373785019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,31,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,31,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,63,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,63,0.02141333371400833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,127,0.014922666052977243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,127,0.021295999487241108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,255,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,255,0.020960000654061634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,511,0.023365333676338196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,511,0.025125332176685333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,1023,0.033610666791598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,1023,0.03369600077470144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,2047,0.05403199791908264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,2047,0.05228800078233083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,4095,0.09522133072217305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,4095,0.09100799759229024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,8191,0.1774239937464396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,8191,0.16500799854596457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,1,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,float16,16383,0.34284265836079914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,1,0.016895999511082966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,3,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,1,128,0,1,float16,fp8,16383,0.3120959997177124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,3,0.01682666689157486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,7,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,7,0.016255999604860943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,15,0.014826666563749313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,15,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,31,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,31,0.017450666675964992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,63,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,63,0.020992000897725422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,127,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,127,0.021162666380405426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,255,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,255,0.021733333667119343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,511,0.02332266668478648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,511,0.023200000325838726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,1023,0.03376533339420954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,1023,0.033626665671666466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,2047,0.053957333167394005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,2047,0.05225066840648651
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,4095,0.09531199932098389
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,4095,0.09077866872151692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,8191,0.17750932772954306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,1,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,8191,0.16547733545303345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,float16,16383,0.3428373336791992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,4,128,0,1,float16,fp8,16383,0.31224000453948975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,3,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,3,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,7,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,15,0.011968000481526056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,31,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,63,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,255,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,511,0.012778667112191519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,511,0.012655999511480331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,1023,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,1023,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,2047,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,2047,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,4095,0.017397332936525345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,4095,0.01746133342385292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,8191,0.019546666493018467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,8191,0.01922133316596349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,float16,16383,0.02319466571013133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,1,0.0671253353357315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,96,8,128,0,1,float16,fp8,16383,0.02143999934196472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,1,0.05801066756248474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,3,0.06844800213972728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,3,0.058304001887639366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,7,0.07030400137106578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,7,0.06186666587988535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,15,0.08267733454704285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,15,0.07640533149242401
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,31,0.08463467160860698
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,31,0.07521066566308339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,63,0.08478400111198425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,63,0.07644799848397572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,127,0.09899200002352397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,127,0.09103467067082723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,255,0.14849600195884705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,255,0.14033066232999167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,511,0.24709866444269815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,511,0.23669334252675375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,1023,0.4472959836324056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,1023,0.4283839861551921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,1,0.06644799808661143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,float16,2047,0.8455573717753092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,1,0.057775999108950295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,1,128,0,1,float16,fp8,2047,0.8115839958190918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,3,0.06870933373769124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,3,0.05894400179386139
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,7,0.06854400038719177
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,7,0.06070933242638906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,15,0.0845973292986552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,15,0.07469333211580913
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,31,0.08459200461705525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,31,0.07599466542402904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,63,0.08475200335184734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,63,0.07657066484292348
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,127,0.09929600358009338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,127,0.09107200304667155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,255,0.14873066544532776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,255,0.14037332932154337
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,511,0.24779733022054037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,511,0.23657600084940592
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,1023,0.4562613169352214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,1023,0.4273973306020101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,1,0.01706133286158244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,1,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,3,0.01682666689157486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,7,0.017018667111794155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,3,0.016997333616018295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,float16,2047,0.8675253391265869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,15,0.017018667111794155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,7,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,15,0.016821333517630894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,31,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,31,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,4,128,0,1,float16,fp8,2047,0.8243520259857178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,63,0.017456000049908955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,63,0.01666133354107539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,127,0.01718933383623759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,255,0.01743999992807706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,127,0.017898666361967724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,255,0.016938666502634685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,511,0.02109866589307785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,511,0.021146667500336964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,1023,0.025445332129796345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,1023,0.03731200098991394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,float16,2047,0.06069866816202799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,96,8,128,0,1,float16,fp8,2047,0.04437333345413208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,1,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,3,0.017184000462293625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,1,0.015664000064134598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,3,0.015477333217859268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,7,0.017845333864291508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,7,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,15,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,15,0.019658666104078293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,31,0.021205333371957142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,31,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,63,0.021253332495689392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,127,0.021183999876181286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,63,0.01793066660563151
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,127,0.01695466662446658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,255,0.023285334308942158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,255,0.019472000499566395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,511,0.03201599915822347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,511,0.027514666318893433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,1023,0.048512001832326256
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,1023,0.04234133164087931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,2047,0.0790293316046397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,2047,0.07261866827805837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,4095,0.14058132966359457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,4095,0.13058132926623026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,8191,0.26523733139038086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,8191,0.24893333514531454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,1,0.018911999960740406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,1,0.015450666348139444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,float16,16383,0.5115466515223185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,3,0.019039999693632126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,1,128,0,1,float16,fp8,16383,0.48361066977183026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,3,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,7,0.01922133316596349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,7,0.015429332852363586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,15,0.019120000302791595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,15,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,31,0.021157334248224895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,31,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,63,0.021642667551835377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,63,0.01718933383623759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,127,0.021546666820844013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,127,0.01727466657757759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,255,0.02346133440732956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,255,0.019296000401178997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,511,0.03192000091075897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,511,0.027637332677841187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,1023,0.04797333478927612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,1023,0.04190933207670847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,2047,0.07916266719500224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,2047,0.0728959987560908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,4095,0.14043733477592468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,4095,0.13247467080752054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,8191,0.2654186685880025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,8191,0.2488266626993815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,1,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,3,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,float16,16383,0.5151040156682333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,3,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,4,128,0,1,float16,fp8,16383,0.4859466552734375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,7,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,7,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,15,0.012682666381200155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,15,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,31,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,63,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,63,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,127,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,127,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,255,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,255,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,511,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,511,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,1023,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,1023,0.013429333766301474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,2047,0.015568000574906668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,2047,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,4095,0.017749333133300144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,4095,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,8191,0.02124800036350886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,8191,0.019440000255902607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,float16,16383,0.039306665460268654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,96,8,128,0,1,float16,fp8,16383,0.02571733295917511
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,1,0.1279093325138092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,1,0.10784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,3,0.13032533725102743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,3,0.11030399799346924
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,7,0.11565867066383362
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,15,0.16085333625475565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,7,0.13361600041389465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,15,0.14416533708572388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,31,0.1612266699473063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,31,0.14442132910092673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,63,0.14432000120480856
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,63,0.1629813313484192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,127,0.191594660282135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,127,0.17510400215784708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,255,0.28837867577870685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,255,0.27163199583689374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,511,0.48260800043741864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,511,0.46134400367736816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,1,0.12776000301043192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,1,0.10731732845306396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,float16,1023,0.8774027029673258
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,3,0.13129599889119467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,3,0.1113866666952769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,1,128,0,1,float16,fp8,1023,0.8356426556905111
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,7,0.11594667037328084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,7,0.13390933473904928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,15,0.16193600495656332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,15,0.14432000120480856
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,31,0.14433599511782327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,31,0.16166399916013083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,63,0.1628320018450419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,63,0.14458666245142618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,127,0.1916373372077942
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,127,0.17602666219075522
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,255,0.29037866989771527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,255,0.27167999744415283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,511,0.4948586622873942
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,511,0.4623039960861206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,1,0.02346666653951009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,float16,1023,0.9043413003285726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,4,128,0,1,float16,fp8,1023,0.8533173402150472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,1,0.023445333043734234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,3,0.0236160010099411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,3,0.023183998962243397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,7,0.023797333240509033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,7,0.023376000424226124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,15,0.024986666937669117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,15,0.021530665457248688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,31,0.023242667317390442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,31,0.023130667706330616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,63,0.022629333039124806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,63,0.021381333470344543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,127,0.023258666197458904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,127,0.02160000056028366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,255,0.022677332162857056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,255,0.021375998854637146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,511,0.03577066709597906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,511,0.028565332293510437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,float16,1023,0.056159997979799904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,96,8,128,0,1,float16,fp8,1023,0.04121600091457367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,1,0.2052746613820394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,1,0.24706665674845377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,3,0.25541333357493085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,3,0.21383466323216757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,7,0.2595840096473694
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,7,0.22446399927139282
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,15,0.31622932354609173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,15,0.2802346746126811
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,31,0.316591997941335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,31,0.2815200090408325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,63,0.3189760049184163
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,63,0.2823999921480815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,127,0.3754986524581909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,127,0.34348265329996747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,1,0.24672534068425497
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,1,0.20588799317677817
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,float16,255,0.5700053373972574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,1,128,0,1,float16,fp8,255,0.5337119897206625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,3,0.2550080021222432
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,3,0.2137653430302938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,7,0.224400003751119
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,7,0.26145599285761517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,15,0.3163680036862691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,15,0.2794826626777649
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,31,0.3163306713104248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,31,0.2823359966278076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,63,0.31817599137624103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,63,0.2834239999453227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,127,0.37590400377909344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,127,0.3436746597290039
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,1,0.034186666210492454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,float16,255,0.5897546609242758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,4,128,0,1,float16,fp8,255,0.5354666709899902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,1,0.033786666889985405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,3,0.03358400116364161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,3,0.03356266766786575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,7,0.035562666753927864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,7,0.03383466601371765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,15,0.033904001116752625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,15,0.034346667428811394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,31,0.035504000882307686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,31,0.03345066557327906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,63,0.03419200082619985
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,63,0.033759998778502144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,127,0.03598399957021078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,127,0.033589333295822144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,float16,255,0.038848000268141426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,96,8,128,0,1,float16,fp8,255,0.033786666889985405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,1,0.4006719986597697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,1,0.4862453142801921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,3,0.5027999877929688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,3,0.41886401176452637
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,7,0.4395093520482381
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,7,0.5127626657485962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,15,0.6238720019658407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,15,0.5526560147603353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,31,0.5540266831715902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,31,0.6252373456954956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,63,0.6294613281885783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,63,0.5577866633733114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,fp8,127,0.678762674331665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,1,128,0,1,float16,float16,127,0.7430666287740072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,1,0.48365867137908936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,1,0.40432532628377277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,3,0.41888535022735596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,3,0.5048266649246216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,7,0.513434648513794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,7,0.4414079984029134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,15,0.6252479950586954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,15,0.5522613525390625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,31,0.5553546746571859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,31,0.6253013213475546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,63,0.6318720181783041
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,63,0.5579946835835775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,float16,127,0.7858453591664633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,1,0.0622026671965917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,1,0.05628266433874766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,3,0.062368000547091164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,4,128,0,1,float16,fp8,127,0.6835359732309977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,3,0.05686933298905691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,7,0.06233599781990051
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,7,0.05649066468079885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,15,0.06163200239340464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,15,0.05764266848564148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,31,0.06170133252938589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,63,0.062234664956728615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,31,0.056847999493281044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,63,0.05773866673310598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,float16,127,0.062650665640831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,1,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,96,8,128,0,1,float16,fp8,127,0.058373332023620605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,3,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,1,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,3,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,7,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,7,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,15,0.01747200017174085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,15,0.017024000485738117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,31,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,31,0.015381333728631338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,63,0.017407999684413273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,63,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,127,0.019248000035683315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,127,0.019482667247454327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,255,0.027653334041436512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,255,0.026309333741664886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,511,0.04249600072701772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,511,0.0421973317861557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,1023,0.0728959987560908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,1023,0.07274666428565979
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,2047,0.13421866297721863
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,2047,0.13211733102798462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,4095,0.25539199511210126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,4095,0.2512320081392924
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,8191,0.49876264731089276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,1,0.01413333291808764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,8191,0.49293335278828937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,1,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,3,0.014954666296641031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,3,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,7,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,7,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,15,0.0170666662355264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,15,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,float16,16383,1.0211520195007324
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,31,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,31,0.01709866647919019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,63,0.016976000120242436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,63,0.017349333812793095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,127,0.019381333142518997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,1,128,0,1,float16,fp8,16383,1.0965759754180908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,127,0.019088000059127808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,255,0.02743999908367793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,255,0.02587199956178665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,511,0.0432640016078949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,511,0.04188799858093262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,1023,0.07285866638024648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,1023,0.0726560006539027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,2047,0.1339040001233419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,2047,0.13185066978136697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,4095,0.25734400749206543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,4095,0.25246934096018475
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,8191,0.5017546812693278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,8191,0.49247467517852783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,3,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,7,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,7,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,15,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,31,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,float16,16383,1.1302826404571533
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,31,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,4,128,0,1,float16,fp8,16383,1.1345919768015544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,63,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,63,0.012725333372751871
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,127,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,127,0.012698666503032049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,255,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,255,0.012752000242471695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,511,0.01351999988158544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,511,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,1023,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,1023,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,2047,0.019237333287795384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,2047,0.017466666797796886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,4095,0.021477334201335907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,4095,0.02161066730817159
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,8191,0.027221334477265675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,8191,0.03958933303753535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,float16,16383,0.06001600126425425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,96,8,128,0,1,float16,fp8,16383,0.04186666508515676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,1,0.9639306863149008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,1,0.7928053538004557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,3,1.000384012858073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,3,0.8282933235168457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,7,1.0192639827728271
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,7,0.8735466798146566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,15,1.0973066488901775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,15,1.2424373626708984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,31,1.2427626450856526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,31,1.1021599769592285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,float16,63,1.2521386941274006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,1,128,0,1,float16,fp8,63,1.1070880095163982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,1,0.9614880084991455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,1,0.7991413275400797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,3,1.0029333432515461
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,3,0.8307092984517416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,7,1.025882641474406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,7,0.8742132981618246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,15,1.09879469871521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,15,1.2503573099772136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,31,1.1106080214182537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,31,1.274282693862915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,fp8,63,1.144490639368693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,4,128,0,1,float16,float16,63,1.3027839660644531
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,1,0.11074666182200114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,1,0.10594133536020915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,3,0.11045866211255391
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,3,0.10437867045402527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,7,0.11159466703732808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,7,0.10562666257222493
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,15,0.11066133777300517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,15,0.10527466734250386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,31,0.11134933431943257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,31,0.10598400235176086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,float16,63,0.11091732978820801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,96,8,128,0,1,float16,fp8,63,0.10540266831715901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,1,2.0360426902770996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,1,1.683802604675293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,3,1.7259626388549805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,3,2.102565288543701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,7,1.8110346794128418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,7,2.1420000394185386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,15,2.540016015370687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,15,2.2462239265441895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,fp8,31,2.255023956298828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,1,128,0,1,float16,float16,31,2.543621381123861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,1,1.750197410583496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,1,2.127386728922526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,3,1.779354731241862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,3,2.1798240343729653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,7,2.216111977895101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,7,1.8532907168070476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,15,2.2733333905537925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,15,2.573802630106608
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,fp8,31,2.2815893491109214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,4,128,0,1,float16,float16,31,2.5773119926452637
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,1,0.2079520026842753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,1,0.19740267594655356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,3,0.20992533365885416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,3,0.197818656762441
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,7,0.20823999245961508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,7,0.1978879968325297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,15,0.20959999163945517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,15,0.1978293259938558
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,1,0.02346666653951009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,3,0.023413332800070446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,1,0.019237333287795384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,3,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,7,0.025472000241279602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,7,0.019493332753578823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,15,0.027066667874654133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,15,0.02327999969323476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,31,0.027077332139015198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,31,0.023520000278949738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,63,0.02749866743882497
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,63,0.025258667767047882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,127,0.03156266609827677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,float16,31,0.2078239917755127
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,127,0.029322666426499683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,96,8,128,0,1,float16,fp8,31,0.1980853279431661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,255,0.04390400151411692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,255,0.042581334710121155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,511,0.0724373310804367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,511,0.07042666773001353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,1023,0.12778133153915405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,1023,0.12396799524625142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,2047,0.2343733310699463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,2047,0.2301386594772339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,4095,0.45396800835927326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,4095,0.44354132811228436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,1,0.019306667149066925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,1,0.023311999936898548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,3,0.02330133318901062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,3,0.020725333442290623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,7,0.023061332603295643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,7,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,float16,8191,0.8914453188578287
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,1,128,0,1,float16,fp8,8191,0.870527982711792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,15,0.02735466758410136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,15,0.02532266577084859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,31,0.02569599946339925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,31,0.025263999899228413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,63,0.027242665489514668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,63,0.025061334172884624
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,127,0.03141866624355316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,127,0.02942933390537898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,255,0.04410133262475332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,255,0.04354133208592733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,511,0.07276799778143565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,511,0.0708000014225642
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,1023,0.12603200475374857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,1023,0.12355200449625652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,2047,0.23643734057744345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,2047,0.23067732652028403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,4095,0.45564266045888263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,4095,0.4432640075683594
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,1,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,1,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,3,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,float16,8191,0.9093653361002604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,3,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,7,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,4,128,0,1,float16,fp8,8191,0.8740373452504476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,7,0.012613333761692047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,15,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,15,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,31,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,63,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,63,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,127,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,255,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,255,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,511,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,511,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,1023,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,1023,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,2047,0.02103466788927714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,2047,0.019248000035683315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,4095,0.03772799919048945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,4095,0.025381334125995636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,float16,8191,0.05840000013510386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,1,0.02771199991305669
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,96,8,128,0,1,float16,fp8,8191,0.04154133299986521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,1,0.027615999182065327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,1,0.023472001155217487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,1,0.023402666052182514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,3,0.027530667682488758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,3,0.02757866680622101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,3,0.02332799881696701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,3,0.02334933231274287
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,7,0.029071999092896778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,7,0.027674667537212372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,7,0.02518933266401291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,7,0.025258667767047882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,15,0.02937600016593933
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,15,0.029130667448043823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,15,0.025311999022960663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,15,0.025258667767047882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,31,0.03403199960788091
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,31,0.03363733241955439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,31,0.029738667110602062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,31,0.029738667110602062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,63,0.0337119996547699
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,63,0.03332799921433131
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,63,0.031178665657838184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,63,0.03025600065787633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,127,0.03385066737731298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,127,0.03366400053103765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,127,0.03127466638882955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,127,0.029989334444204967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,255,0.03393599887688955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,255,0.03125333289305369
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,255,0.03579200059175491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,255,0.04013866682847341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,511,0.033813332517941795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,511,0.030346666773160298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,511,0.05629333357016245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,511,0.05375466744105021
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,1023,0.03012266755104065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,1023,0.034714666505654655
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,1023,0.09058133761088054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,1023,0.08694400389989217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,2047,0.03331200033426285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,2047,0.03137599925200144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,2047,0.16103466351826987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,2047,0.1567306617895762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,4095,0.033471999069054924
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,4095,0.30004266897837323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,4095,0.030778666337331135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,float16,8191,0.03340800106525421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,4095,0.29598933458328247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,1,0.02754666656255722
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,128,1,float16,fp8,8191,0.03001066545645396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,1,0.027722666660944622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,1,0.02462933212518692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,float16,8191,0.5789493322372437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,1,0.024282666544119518
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,64,0,1,float16,fp8,8191,0.5727200110753378
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,3,0.02758399893840154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,3,0.027647999425729115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,3,0.023365333676338196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,3,0.024031999210516613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,7,0.029189333319664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,7,0.029279999434947968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,7,0.025349333882331848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,7,0.025034666061401367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,15,0.029232000311215717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,15,0.029338667790095013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,15,0.02516266703605652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,15,0.025242666403452556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,31,0.03387733300526937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,31,0.03342399994532267
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,31,0.02924799919128418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,31,0.030218665798505146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,63,0.03344533344109853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,63,0.03339199970165888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,63,0.030805334448814392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,63,0.029872000217437744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,127,0.03339199970165888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,127,0.03349866718053818
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,127,0.02958933264017105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,127,0.029578665892283123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,255,0.033717334270477295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,255,0.039605334401130676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,255,0.029317334294319153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,255,0.03562666724125544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,511,0.03369066615899404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,511,0.056015998125076294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,511,0.029504001140594482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,511,0.053930665055910744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,1023,0.03342399994532267
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,1023,0.09081600109736125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,1023,0.031146667897701263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,1023,0.086709330479304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,2047,0.03367999941110611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,2047,0.1604746679464976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,2047,0.029605334003766377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,2047,0.15678933262825012
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,4095,0.033759998778502144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,4095,0.2993386586507161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,4095,0.02994133283694585
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,4095,0.2957013249397278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,float16,8191,0.033344000577926636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,128,1,float16,fp8,8191,0.03143466760714849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,float16,8191,0.5820693174997965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,1,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,1,0.012634667257467905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,3,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,64,0,1,float16,fp8,8191,0.5724053382873535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,3,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,3,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,7,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,7,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,7,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,7,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,15,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,15,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,31,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,31,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,31,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,63,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,63,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,63,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,63,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,127,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,127,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,127,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,255,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,127,0.012213333199421564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,255,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,255,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,511,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,511,0.011722666521867117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,511,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,511,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,1023,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,1023,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,1023,0.01482133318980535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,1023,0.013936000565687815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,2047,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,2047,0.01951466624935468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,2047,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,2047,0.01942933350801468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,4095,0.015306666493415833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,4095,0.025605333348115284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,4095,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,4095,0.023792001108328503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,float16,8191,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,float16,8191,0.0420959989229838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,128,1,float16,fp8,8191,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,1,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,1,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,64,0,1,float16,fp8,8191,0.03595733394225439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,1,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,1,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,3,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,3,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,3,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,3,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,7,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,7,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,7,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,7,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,15,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,15,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,15,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,15,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,31,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,31,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,31,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,31,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,63,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,63,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,63,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,63,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,127,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,127,0.012741333494583765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,127,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,127,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,255,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,255,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,255,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,255,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,511,0.012752000242471695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,511,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,511,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,511,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,1023,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,1023,0.0170666662355264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,1023,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,1023,0.015402667224407196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,2047,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,2047,0.023418667415777843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,2047,0.015413332730531693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,2047,0.023242667317390442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,4095,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,4095,0.039408000806967415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,4095,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,4095,0.0315786674618721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,float16,8191,0.015685333559910457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,float16,8191,0.060191998879114784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,128,1,float16,fp8,8191,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,1,0.008826666822036108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,1,0.00922133338948091
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,64,0,1,float16,fp8,8191,0.05220800141493479
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,1,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,1,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,3,0.00922133338948091
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,3,0.008832000195980072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,3,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,3,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,7,0.009136000027259191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,7,0.009290666629870733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,15,0.009183999771873156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,15,0.00919999989370505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,15,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,31,0.00921066664159298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,31,0.00890666681031386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,63,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,63,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,63,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,63,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,127,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,127,0.010543999572594961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,127,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,127,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,255,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,255,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,255,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,255,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,511,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,511,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,511,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,1023,0.019205333044131596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,511,0.015615999698638916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,1023,0.021594665944576263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,1023,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,1023,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,2047,0.020031999796628952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,2047,0.025754667818546295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,2047,0.02317333221435547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,2047,0.029301332930723827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,4095,0.0336053321758906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,4095,0.02312533309062322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,4095,0.019733333339293797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,4095,0.037530665596326195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,8191,0.018960000326236088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,8191,0.050442665815353394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,8191,0.023232000569502514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,8191,0.05009066561857859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,16383,0.019333332777023315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,16383,0.07922666768232982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,16383,0.02327999969323476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,16383,0.07710400223731995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,float16,32767,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,float16,32767,0.1406826674938202
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,1,0.009141333401203156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,128,1,float16,fp8,32767,0.023520000278949738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,1,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,1,0.00916800027092298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,64,0,1,float16,fp8,32767,0.12690132856369019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,1,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,3,0.009365333244204521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,3,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,3,0.00943999985853831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,7,0.009445333232482275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,7,0.009712000067035357
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,7,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,7,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,15,0.009514666472872099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,15,0.009445333232482275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,31,0.010005333150426546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,31,0.010144000252087912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,31,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,31,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,63,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,63,0.01190399999419848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,127,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,127,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,127,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,255,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,255,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,255,0.01349866638580958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,511,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,255,0.013445333888133367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,511,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,511,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,511,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,1023,0.02146133283774058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,1023,0.018853332847356796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,1023,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,1023,0.01732800031701724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,2047,0.01911466692884763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,2047,0.027136000494162243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,2047,0.023056000471115112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,2047,0.029839999973773956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,4095,0.01956266661485036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,4095,0.03356266766786575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,4095,0.023029332359631855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,4095,0.03760000069936117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,8191,0.018954666952292126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,8191,0.023200000325838726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,8191,0.04938133557637533
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,8191,0.04987733562787374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,16383,0.01883200059334437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,16383,0.07933866480986278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,16383,0.023029332359631855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,16383,0.07497066756089528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,float16,32767,0.019968000551064808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,float16,32767,0.14014400045077005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,1,0.010458666831254959
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,128,1,float16,fp8,32767,0.02481066683928172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,1,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,1,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,1,0.011802667131026586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,64,0,1,float16,fp8,32767,0.12622400124867758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,3,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,3,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,3,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,7,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,7,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,7,0.011760000139474869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,15,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,15,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,15,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,31,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,15,0.012458667159080505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,31,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,31,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,31,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,63,0.012367999802033106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,63,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,127,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,127,0.012341332932313284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,127,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,255,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,255,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,511,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,511,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,511,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,511,0.013573333621025085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,1023,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,1023,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,1023,0.011781333635250727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,1023,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,2047,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,2047,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,2047,0.012682666381200155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,2047,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,4095,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,4095,0.013850666582584381
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,4095,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,4095,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,8191,0.011776000261306763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,8191,0.017093333105246227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,8191,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,8191,0.017845333864291508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,16383,0.01905599981546402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,16383,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,16383,0.011754666765530905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,16383,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,float16,32767,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,128,1,float16,fp8,32767,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,float16,32767,0.021359999974568684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,1,0.010565333068370819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,1,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,64,0,1,float16,fp8,32767,0.01956266661485036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,3,0.010581333190202713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,3,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,3,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,3,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,7,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,7,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,7,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,7,0.011600000162919363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,15,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,15,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,15,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,15,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,31,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,31,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,31,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,63,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,63,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,63,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,127,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,127,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,127,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,255,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,255,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,255,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,511,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,255,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,511,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,511,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,511,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,1023,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,1023,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,1023,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,1023,0.01246400053302447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,2047,0.012234666695197424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,2047,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,2047,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,2047,0.012768000364303589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,4095,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,4095,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,4095,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,4095,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,8191,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,8191,0.01598400001724561
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,8191,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,8191,0.017898666361967724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,16383,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,16383,0.01759999990463257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,16383,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,16383,0.017605333278576534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,float16,32767,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,1,0.009578666960199675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,128,1,float16,fp8,32767,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,float16,32767,0.018933333456516266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,1,0.009317333499590555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,64,0,1,float16,fp8,32767,0.019237333287795384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,3,0.009365333244204521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,3,0.008853333070874214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,3,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,7,0.009050666665037474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,7,0.009290666629870733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,15,0.009125333279371262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,7,0.01062400018175443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,15,0.008869333192706108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,15,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,15,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,31,0.00921066664159298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,31,0.008821333448092142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,31,0.010565333068370819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,63,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,63,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,63,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,127,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,127,0.010543999572594961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,127,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,127,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,255,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,255,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,255,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,511,0.019050666441520054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,511,0.021386665602525074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,511,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,511,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,1023,0.021221332252025604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,1023,0.025173333783944447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,1023,0.0233599990606308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,1023,0.02755733331044515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,2047,0.020362666497627895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,2047,0.029290666182835896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,2047,0.02514133354028066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,2047,0.03181866556406021
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,4095,0.01934933289885521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,4095,0.0421013335386912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,4095,0.02460266649723053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,4095,0.04219200213750204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,8191,0.019802667200565338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,8191,0.06444266438484192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,8191,0.023445333043734234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,8191,0.06211199859778086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,16383,0.019285333653291065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,16383,0.11154666543006897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,16383,0.02332799881696701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,16383,0.10102933645248413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,float16,32767,0.019482667247454327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,128,1,float16,fp8,32767,0.023386667172114056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,float16,32767,0.20390933752059937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,1,0.009663999701539675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,1,0.009743999689817429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,1,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,3,0.009285333255926767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,3,0.008997333546479544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,64,0,1,float16,fp8,32767,0.1772800087928772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,3,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,7,0.010005333150426546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,7,0.009429333110650381
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,7,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,7,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,15,0.009418666362762451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,15,0.008896000062425932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,31,0.010565333068370819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,31,0.009663999701539675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,31,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,31,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,63,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,63,0.010677333921194077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,63,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,127,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,127,0.010464000205198923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,127,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,127,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,255,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,255,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,255,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,255,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,511,0.019578666736682255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,511,0.02186666677395503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,511,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,511,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,1023,0.020069333414236706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,1023,0.025120000044504803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,1023,0.023472001155217487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,1023,0.027429332335789997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,2047,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,2047,0.029520000020662945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,2047,0.023898666103680927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,2047,0.031498665610949196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,4095,0.021040000021457672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,4095,0.04187199970086416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,4095,0.02974933385848999
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,4095,0.04238933324813843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,8191,0.020762667059898376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,8191,0.06442666550477345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,8191,0.02481599897146225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,8191,0.06229333579540253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,16383,0.019509332875410717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,16383,0.1114026705423991
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,16383,0.0252960001428922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,16383,0.10147200028101604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,float16,32767,0.021738665799299877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,float16,32767,0.20333866278330484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,1,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,128,1,float16,fp8,32767,0.02369066576162974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,1,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,1,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,1,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,64,0,1,float16,fp8,32767,0.1774186690648397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,3,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,3,0.0116799995303154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,3,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,3,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,7,0.011578666667143503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,7,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,7,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,7,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,15,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,15,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,15,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,15,0.011936000237862269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,31,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,31,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,31,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,31,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,63,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,63,0.011829332758982977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,63,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,63,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,127,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,127,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,127,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,127,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,255,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,255,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,255,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,255,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,511,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,511,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,511,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,511,0.012309333930412928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,1023,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,1023,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,1023,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,1023,0.011706666400035223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,2047,0.011920000116030375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,2047,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,2047,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,2047,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,4095,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,4095,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,4095,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,4095,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,8191,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,8191,0.017093333105246227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,8191,0.011989332735538483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,8191,0.017290666699409485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,16383,0.018954666952292126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,16383,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,16383,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,16383,0.01886933296918869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,float16,32767,0.01257066677014033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,float16,32767,0.021018666525681812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,1,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,128,1,float16,fp8,32767,0.012671999633312225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,1,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,1,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,1,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,3,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,3,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,64,0,1,float16,fp8,32767,0.021002667645613354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,3,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,7,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,7,0.012122667084137598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,7,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,15,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,15,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,15,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,31,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,31,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,31,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,63,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,63,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,63,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,127,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,127,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,127,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,127,0.010656000425418219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,255,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,255,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,255,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,255,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,511,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,511,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,511,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,1023,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,1023,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,1023,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,1023,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,2047,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,2047,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,2047,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,2047,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,4095,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,4095,0.015392000476519266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,4095,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,4095,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,8191,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,8191,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,8191,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,8191,0.01553600033124288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,16383,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,16383,0.01951466624935468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,16383,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,16383,0.01926933353145917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,float16,32767,0.011621333658695221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,float16,32767,0.023919999599456787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,128,1,float16,fp8,32767,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,1,0.047930667797724404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,64,0,1,float16,fp8,32767,0.02207999924818675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,1,0.04011733333269755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,1,0.047882666190465294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,1,0.03963200002908707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,3,0.047872001926104225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,3,0.047968000173568726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,3,0.039621333281199135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,3,0.03974399964014689
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,7,0.04807466765244802
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,7,0.048325334986050926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,7,0.04182399809360504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,7,0.04154133299986521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,15,0.05012799799442291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,15,0.05011733373006185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,15,0.042693331837654114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,15,0.04185600082079569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,31,0.05820799867312113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,31,0.06006399790445963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,31,0.051882664362589516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,31,0.051829333106676735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,63,0.0601440022389094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,63,0.060090666015942894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,63,0.05190399785836538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,63,0.051962668697039284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,127,0.052255998055140175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,127,0.060346667965253196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,127,0.06020266811052958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,127,0.05223466455936432
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,255,0.06038933495680491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,255,0.07026133437951405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,255,0.05197333296140035
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,255,0.06262933214505513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,511,0.06050133208433787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,511,0.10314666231473286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,511,0.05376000205675761
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,511,0.09679466485977173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,1023,0.06011733412742615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,1023,0.05385600030422211
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,1023,0.16749866803487143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,1023,0.16077867150306702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,2047,0.060133333007494606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,2047,0.2999626596768697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,2047,0.05303466816743215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,float16,4095,0.060309335589408875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,2047,0.29209067424138385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,1,0.04788800080617269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,128,1,float16,fp8,4095,0.05283733208974203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,1,0.04775466521581014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,float16,4095,0.5662506818771362
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,1,0.04030933231115341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,1,0.0396373321612676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,64,0,1,float16,fp8,4095,0.5577226479848226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,3,0.048154667019844055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,3,0.048341333866119385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,3,0.03988266736268997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,3,0.03985599925120672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,7,0.049813335140546165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,7,0.04153066625197729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,7,0.04939199984073639
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,7,0.041482667128245033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,15,0.0503359983364741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,15,0.04982399940490723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,15,0.04207466542720795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,15,0.042597333590189614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,31,0.06005333364009857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,31,0.05811200042565664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,31,0.05212266743183136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,31,0.052298665046691895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,63,0.059445331494013466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,63,0.06022400160630544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,63,0.052709331115086876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,63,0.05228800078233083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,127,0.06027733286221822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,127,0.06025066475073496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,127,0.05201066533724467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,127,0.052522664268811546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,255,0.06005333364009857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,255,0.0703359991312027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,255,0.05258666475613912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,255,0.062319998939832054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,511,0.060122668743133545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,511,0.10326400399208069
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,511,0.05212800204753876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,511,0.09490133325258891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,1023,0.060218666990598045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,1023,0.052042668064435325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,1023,0.16726932922999063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,1023,0.1607360045115153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,2047,0.06027733286221822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,2047,0.052298665046691895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,2047,0.30002667506535846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,2047,0.2935733397801717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,float16,4095,0.0599839985370636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,128,1,float16,fp8,4095,0.05253866811593374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,float16,4095,0.5728799899419149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,1,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,1,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,1,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,64,0,1,float16,fp8,4095,0.5571306546529134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,1,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,3,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,3,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,3,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,3,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,7,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,7,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,7,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,7,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,15,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,15,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,15,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,15,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,31,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,31,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,31,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,31,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,63,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,63,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,63,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,63,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,127,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,127,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,127,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,127,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,255,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,255,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,255,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,255,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,511,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,511,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,511,0.01350933313369751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,511,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,1023,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,1023,0.013562666873137156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,1023,0.01728533332546552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,1023,0.018218666315078735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,2047,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,2047,0.027056001126766205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,2047,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,2047,0.026133333643277485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,float16,4095,0.01628799984852473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,float16,4095,0.041936000188191734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,128,1,float16,fp8,4095,0.016938666502634685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,1,0.014922666052977243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,1,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,1,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,1,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,64,0,1,float16,fp8,4095,0.03589333345492681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,3,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,3,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,3,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,7,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,3,0.015482666591803232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,7,0.014720000326633453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,7,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,7,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,15,0.015520000209410986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,15,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,15,0.014837333311637243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,15,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,31,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,31,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,31,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,31,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,63,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,63,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,63,0.015333333363135656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,127,0.013738666971524557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,63,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,127,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,127,0.015402667224407196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,127,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,255,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,255,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,255,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,255,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,511,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,511,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,511,0.015423999478419622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,511,0.01687466725707054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,1023,0.015423999478419622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,1023,0.02250666668017705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,1023,0.015114666273196539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,1023,0.021802666286627453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,2047,0.017845333864291508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,2047,0.039701332648595176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,2047,0.017743999759356182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,2047,0.03372266640265783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,float16,4095,0.01743999992807706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,float16,4095,0.061103999614715576
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,128,1,float16,fp8,4095,0.019199999670187633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,1,0.01423466702302297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,1,0.014906667172908783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,1,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,64,0,1,float16,fp8,4095,0.053930665055910744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,1,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,3,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,3,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,3,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,7,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,7,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,7,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,15,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,15,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,15,0.010602666685978571
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,15,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,31,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,31,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,31,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,63,0.017114666601022083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,63,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,63,0.01704000060757001
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,63,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,127,0.016821333517630894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,127,0.016938666502634685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,127,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,127,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,255,0.01682666689157486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,255,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,255,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,255,0.013552000125249227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,511,0.01602666700879733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,511,0.01749333366751671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,511,0.013408000270525614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,511,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,1023,0.016496000190575916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,1023,0.023647998770078022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,1023,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,1023,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,2047,0.015306666493415833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,2047,0.03365333378314972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,2047,0.012661332885424295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,2047,0.029605334003766377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,4095,0.01682666689157486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,4095,0.05435200035572052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,4095,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,4095,0.04982399940490723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,8191,0.01603200038274129
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,8191,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,8191,0.0953653355439504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,8191,0.0909493366877238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,16383,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,16383,0.1791200041770935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,16383,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,16383,0.17100799083709717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,float16,32767,0.015706667055686314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,1,0.013408000270525614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,128,1,float16,fp8,32767,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,1,0.013482666263977686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,float16,32767,0.3429119984308879
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,1,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,64,0,1,float16,fp8,32767,0.3471946716308594
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,3,0.014544000228246054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,3,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,3,0.010581333190202713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,3,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,7,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,7,0.014549333602190018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,7,0.010469333579142889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,15,0.014885333677132925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,15,0.013989333063364029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,15,0.010608000059922537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,15,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,31,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,31,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,31,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,63,0.016864000509182613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,63,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,63,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,63,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,127,0.016143999993801117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,127,0.016293333222468693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,127,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,127,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,255,0.01599466676513354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,255,0.017008000363906223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,255,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,255,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,511,0.01685333376129468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,511,0.018730666488409042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,511,0.012709333250919977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,511,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,1023,0.01714666684468587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,1023,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,1023,0.02317333221435547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,1023,0.019189332922299702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,2047,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,2047,0.03384533276160558
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,2047,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,2047,0.029493334392706554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,4095,0.01602666700879733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,4095,0.054101333022117615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,4095,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,4095,0.050330668687820435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,8191,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,8191,0.09542399644851685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,8191,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,8191,0.09098133444786072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,16383,0.015989333391189575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,16383,0.17919466892878214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,16383,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,16383,0.17031999429066977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,float16,32767,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,float16,32767,0.3452800114949544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,128,1,float16,fp8,32767,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,1,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,1,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,1,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,3,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,64,0,1,float16,fp8,32767,0.3469760020573934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,3,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,3,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,3,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,7,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,7,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,7,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,7,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,15,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,15,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,15,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,31,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,31,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,31,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,31,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,63,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,63,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,63,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,63,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,127,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,127,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,127,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,255,0.012319999436537424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,255,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,511,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,511,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,511,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,511,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,1023,0.011701333026091257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,1023,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,1023,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,2047,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,1023,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,2047,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,2047,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,2047,0.015637333194414776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,4095,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,4095,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,4095,0.0136266661187013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,4095,0.01716800034046173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,8191,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,8191,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,8191,0.01877333347996076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,8191,0.01403733342885971
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,16383,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,16383,0.021216000119845074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,16383,0.013674666484196981
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,16383,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,float16,32767,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,float16,32767,0.027263998985290527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,1,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,128,1,float16,fp8,32767,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,1,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,1,0.011610666910807291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,64,0,1,float16,fp8,32767,0.025455998877684276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,3,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,3,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,7,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,7,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,7,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,15,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,31,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,31,0.010522666076819101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,31,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,63,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,63,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,63,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,127,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,127,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,127,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,255,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,255,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,255,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,511,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,511,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,511,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,511,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,1023,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,1023,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,1023,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,1023,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,2047,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,2047,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,2047,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,2047,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,4095,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,4095,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,4095,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,4095,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,8191,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,8191,0.019317333896954853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,8191,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,8191,0.017263999829689663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,16383,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,16383,0.023418667415777843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,16383,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,16383,0.023232000569502514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,float16,32767,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,float16,32767,0.03755199909210205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,128,1,float16,fp8,32767,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,1,0.08911466598510742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,1,0.07246933380762736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,64,0,1,float16,fp8,32767,0.03128000100453695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,1,0.08998933434486389
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,1,0.07281599938869476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,3,0.08902933200200398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,3,0.08885332942008972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,3,0.07339199880758922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,3,0.07250666618347168
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,7,0.09122666716575623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,7,0.09091200431187947
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,7,0.07448533177375793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,15,0.09303466478983562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,15,0.09285333752632141
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,7,0.07699200014273326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,15,0.07875200112660725
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,15,0.07830399771531422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,31,0.1113866666952769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,31,0.11236266295115153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,31,0.09723732868830363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,31,0.09727999567985535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,63,0.11319466431935628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,63,0.09898133079210918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,63,0.0990826686223348
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,63,0.1141919990380605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,127,0.11331199606259663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,127,0.11351999640464783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,127,0.09894933303197224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,127,0.09970666964848836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,255,0.11367467045783997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,255,0.13378133376439413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,255,0.09922132889429729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,511,0.11363200346628825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,255,0.11983999609947205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,511,0.09930666287740071
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,511,0.19774399201075235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,511,0.18567999203999838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,1023,0.11388799548149109
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,1023,0.32657066980997723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,1023,0.09963732957839966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,1023,0.3144906759262085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,float16,2047,0.11424533526102702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,128,1,float16,fp8,2047,0.09974400202433269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,float16,2047,0.5912533203760783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,1,0.08905599514643352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,1,0.08925333619117737
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,64,0,1,float16,fp8,2047,0.576533317565918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,1,0.07308266560236613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,1,0.07309333483378093
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,3,0.0895306666692098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,3,0.08917867143948872
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,3,0.07301866511503856
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,3,0.07311466832955678
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,7,0.09147199988365173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,7,0.09150933225949605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,7,0.07621333499749501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,7,0.07572799921035767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,15,0.09339732925097148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,15,0.09324799974759419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,15,0.07871466875076294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,15,0.07922133306662242
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,31,0.11187199751536052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,31,0.1118239959081014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,31,0.09774399797121684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,31,0.09898133079210918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,63,0.11343999703725179
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,63,0.11327466368675232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,63,0.09949866930643718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,63,0.09929600358009338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,127,0.11414399743080139
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,127,0.09918399651845296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,127,0.11382933457692464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,127,0.09938666224479675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,255,0.11371733744939168
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,255,0.13403733571370444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,255,0.09919466574986775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,255,0.11961600184440613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,511,0.11402133107185364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,511,0.09935466448465984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,511,0.1973759929339091
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,511,0.18521066506703696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,1023,0.09940800070762634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,1023,0.11425600449244182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,1023,0.3266666730244954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,1023,0.31435734033584595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,float16,2047,0.11402133107185364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,128,1,float16,fp8,2047,0.09937600294748943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,1,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,1,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,float16,2047,0.6010880072911581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,64,0,1,float16,fp8,2047,0.5761333306630453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,1,0.015450666348139444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,1,0.015834666788578033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,3,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,3,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,3,0.015418666104475657
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,3,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,7,0.015279999623696009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,7,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,7,0.015626666446526844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,7,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,15,0.015589332828919092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,15,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,15,0.015370666980743408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,15,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,31,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,31,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,31,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,31,0.01552533358335495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,63,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,63,0.015573333948850632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,63,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,63,0.015546667079130808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,127,0.015482666591803232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,127,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,127,0.01571200042963028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,127,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,255,0.01588800052801768
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,255,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,255,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,255,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,511,0.015834666788578033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,511,0.01926400015751521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,511,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,511,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,1023,0.016165333489576977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,1023,0.025413334369659424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,1023,0.01681600014368693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,1023,0.023610666394233704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,float16,2047,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,float16,2047,0.04399466514587402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,128,1,float16,fp8,2047,0.01916266605257988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,1,0.017621333400408428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,64,0,1,float16,fp8,2047,0.0366239994764328
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,1,0.01720533271630605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,1,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,3,0.01773333301146825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,1,0.017535999417304993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,3,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,3,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,3,0.017642666896184284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,7,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,7,0.018112000077962875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,7,0.017375999440749485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,7,0.018730666488409042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,15,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,15,0.017664000391960144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,15,0.018933333456516266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,15,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,31,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,31,0.01899733394384384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,31,0.019648000597953796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,31,0.019189332922299702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,63,0.01903466631968816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,63,0.018800000349680584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,63,0.019189332922299702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,63,0.017653333644072216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,127,0.01887999971707662
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,127,0.019215999792019527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,127,0.018112000077962875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,255,0.019359999646743137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,127,0.018976000448067982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,255,0.017914666483799618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,255,0.019146667172511418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,255,0.019029332945744198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,511,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,511,0.019013332823912304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,511,0.023200000325838726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,511,0.023215999205907185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,1023,0.019071999937295914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,1023,0.0355679988861084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,1023,0.019082666685183842
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,1023,0.029765332738558452
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,float16,2047,0.02162133405605952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,float16,2047,0.05760000149408976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,128,1,float16,fp8,2047,0.02111999938885371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,64,0,1,float16,fp8,2047,0.05053333441416422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,1,0.17102932929992676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,1,0.1710240046183268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,1,0.13823466499646506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,1,0.13833066821098328
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,3,0.17088532447814941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,3,0.17063466707865396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,3,0.13807466626167297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,3,0.1383039951324463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,7,0.1760480006535848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,7,0.175653338432312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,7,0.14288533727327982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,7,0.1443893313407898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,15,0.17907732725143433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,15,0.17988266547520956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,15,0.14854400356610617
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,15,0.1506186624368032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,31,0.216213325659434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,31,0.18938666582107544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,31,0.1895093321800232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,31,0.21619200706481934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,63,0.2181653380393982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,63,0.19155200322469076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,63,0.21809067328770956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,63,0.1914880077044169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,127,0.21993066867192587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,127,0.22020266453425089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,127,0.19163199265797934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,127,0.19166400035222372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,255,0.21991467475891113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,255,0.19165333112080893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,255,0.2587786714235942
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,255,0.23064533869425455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,float16,511,0.22022932767868042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,128,1,float16,fp8,511,0.19149333238601685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,1,0.1706719994544983
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,float16,511,0.3861813147862752
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,64,0,1,float16,fp8,511,0.3615093231201172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,1,0.17071467638015747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,1,0.1383039951324463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,1,0.13829867045084634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,3,0.17113065719604492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,3,0.17083199818929037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,3,0.13819733262062073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,3,0.13827733198801676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,7,0.17525867621103922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,7,0.14410133163134256
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,7,0.17531200249989828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,7,0.1442026694615682
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,15,0.1791413426399231
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,15,0.17916266123453775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,15,0.15050666530927023
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,15,0.1492959956328074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,31,0.21592533588409424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,31,0.21596266825993857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,31,0.1893813411394755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,31,0.18954666455586752
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,63,0.21804799636205038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,63,0.1911733349164327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,63,0.21801066398620605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,63,0.19099734226862589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,127,0.2198293407758077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,127,0.2198293407758077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,127,0.19163199265797934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,127,0.1911946733792623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,255,0.22004799048105875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,255,0.25857067108154297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,255,0.19128533204396567
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,255,0.2320479949315389
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,float16,511,0.22036800781885782
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,128,1,float16,fp8,511,0.19138665994008383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,float16,511,0.3879733482996623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,1,0.01923199991385142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,1,0.021040000021457672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,64,0,1,float16,fp8,511,0.3614453474680583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,1,0.019498666127522785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,1,0.021295999487241108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,3,0.01929066702723503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,3,0.019445333629846573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,3,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,3,0.01952533299724261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,7,0.019285333653291065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,7,0.01942933350801468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,7,0.01932799940307935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,7,0.020928000410397846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,15,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,15,0.019487999379634857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,15,0.021290667355060577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,15,0.019589333484570186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,31,0.01964266722400983
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,31,0.021151999632517498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,31,0.019306667149066925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,31,0.021013334393501282
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,63,0.021168000996112823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,63,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,63,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,63,0.02094399929046631
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,127,0.020725333442290623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,127,0.019440000255902607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,127,0.021183999876181286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,255,0.021130666136741638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,255,0.019440000255902607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,127,0.021920000513394673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,255,0.02149333308140437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,255,0.021295999487241108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,float16,511,0.02124800036350886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,float16,511,0.026341333985328674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,128,1,float16,fp8,511,0.02272533377011617
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,64,0,1,float16,fp8,511,0.02548266698916753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,1,0.026682667434215546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,1,0.025455998877684276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,1,0.02678400029738744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,1,0.02741333345572154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,3,0.02532800038655599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,3,0.025674665967623394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,3,0.026261332134405773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,7,0.02550400048494339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,7,0.025616000096003216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,3,0.02743999908367793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,7,0.026517334083716076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,7,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,15,0.025546667476495106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,15,0.027664000789324444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,15,0.025285333395004272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,15,0.025258667767047882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,31,0.02733866622050603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,31,0.027530667682488758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,31,0.02553066611289978
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,31,0.02521066615978877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,63,0.027061333258946735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,63,0.027248000105222065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,63,0.025701334079106648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,63,0.02550933261712392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,127,0.027237333357334137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,127,0.02738133321205775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,127,0.025466665625572205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,127,0.02516799916823705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,255,0.027642667293548584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,255,0.025749333202838898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,255,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,255,0.025839999318122864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,float16,511,0.02718399961789449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,float16,511,0.03770133356253306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,128,1,float16,fp8,511,0.027072000006834667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,64,0,1,float16,fp8,511,0.033786666889985405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,1,0.3328319986661275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,1,0.26708267132441205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,1,0.33267199993133545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,1,0.26746666431427
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,3,0.333024005095164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,3,0.332586665948232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,3,0.26793066660563153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,3,0.26781866947809857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,7,0.3441173235575358
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,7,0.27929067611694336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,7,0.3429226477940877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,7,0.27949867645899457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,15,0.3510133425394694
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,15,0.2916906674702962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,15,0.3508106470108032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,15,0.29154666264851886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,31,0.42466668287913006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,31,0.42481064796447754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,31,0.37142399946848553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,31,0.37147732575734455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,63,0.4291306734085083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,63,0.42903467019399005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,63,0.37352534135182697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,63,0.3739519913991292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,127,0.43225598335266113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,127,0.43274664878845215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,127,0.37622400124867755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,127,0.37615466117858887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,float16,255,0.43323198954264325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,float16,255,0.5089653333028158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,128,1,float16,fp8,255,0.3760266701380412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,1,0.33295466502507526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,64,0,1,float16,fp8,255,0.45482667287190753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,1,0.3330399990081787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,1,0.2692799965540568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,1,0.2691306670506795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,3,0.3332479993502299
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,3,0.3352266550064087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,3,0.2691253423690796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,3,0.26925865809122723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,7,0.34488534927368164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,7,0.27964266141255695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,7,0.2793920040130615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,7,0.3454293409983317
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,15,0.35074134667714435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,15,0.29157867034276325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,15,0.35101866722106934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,15,0.29224000374476117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,31,0.42506666978200275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,31,0.37118931611378986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,31,0.42477866013844806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,31,0.3715999921162923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,63,0.42933865388234455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,63,0.3738826513290405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,63,0.4294613202412923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,63,0.3753066857655843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,127,0.4328213135401408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,127,0.3760853211085002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,127,0.43133866786956787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,127,0.37642133235931396
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,float16,255,0.4337173302968343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,128,1,float16,fp8,255,0.3757280111312866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,float16,255,0.5087999900182089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,1,0.029834667841593426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,64,0,1,float16,fp8,255,0.4556533495585124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,1,0.029343999922275543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,1,0.03160000095764796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,1,0.03169066707293192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,3,0.029535998900731403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,3,0.02939733366171519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,3,0.03160000095764796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,3,0.03128000100453695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,7,0.029631999631722767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,7,0.02939733366171519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,7,0.031328000128269196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,7,0.03148266673088074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,15,0.029813334345817566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,15,0.029391999046007793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,15,0.03146666785081228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,15,0.031199999153614044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,31,0.029674666623274486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,31,0.029743999242782593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,31,0.031082667410373688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,31,0.03150933235883713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,63,0.02945599953333537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,63,0.029450667401154835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,63,0.03169599920511246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,63,0.03136533250411352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,127,0.029525332152843475
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,127,0.029637334247430164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,127,0.0312266672650973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,127,0.03148266673088074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,float16,255,0.03141866624355316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,float16,255,0.029680001238981884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,128,1,float16,fp8,255,0.03173866619666418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,64,0,1,float16,fp8,255,0.03138133386770884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,1,0.04172799984614054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,1,0.04197866717974345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,1,0.041637333730856575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,1,0.04190933207670847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,3,0.042165334026018776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,3,0.041850666205088295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,3,0.042026668787002563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,3,0.04164266586303711
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,7,0.04170133173465729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,7,0.04182933270931244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,7,0.04227200150489807
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,7,0.04180799921353658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,15,0.04162666698296865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,15,0.04186666508515676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,15,0.04185600082079569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,15,0.041877334316571556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,31,0.041893333196640015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,31,0.042165334026018776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,31,0.04201599955558777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,31,0.041797334949175514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,63,0.04178666571776072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,63,0.04248000184694926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,63,0.042165334026018776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,63,0.042026668787002563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,127,0.041936000188191734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,127,0.04181333382924398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,127,0.04262933135032654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,127,0.04228800038496653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,float16,255,0.04381866753101349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,float16,255,0.04479999840259552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,1,0.013525333255529404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,128,1,float16,fp8,255,0.04197866717974345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,1,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,64,0,1,float16,fp8,255,0.04173333446184794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,1,0.015610666324694952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,1,0.015114666273196539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,3,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,3,0.013781332721312841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,3,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,3,0.015589332828919092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,7,0.014746667196353277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,7,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,7,0.01540800059835116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,7,0.015882667154073715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,15,0.014677333335081736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,15,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,15,0.015520000209410986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,15,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,31,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,31,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,31,0.01693333312869072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,31,0.017887999614079792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,63,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,63,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,63,0.0210506667693456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,63,0.021322667598724365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,127,0.015423999478419622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,127,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,127,0.021253332495689392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,127,0.02120000123977661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,255,0.01716800034046173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,255,0.015578666081031164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,255,0.021055998901526134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,255,0.021274665991465252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,511,0.015306666493415833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,511,0.021850667893886566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,511,0.021157334248224895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,511,0.023061332603295643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,1023,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,1023,0.031888000667095184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,1023,0.019738666713237762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,1023,0.03408000121514002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,2047,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,2047,0.05258133510748545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,2047,0.021157334248224895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,2047,0.053898667295773826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,4095,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,4095,0.021226666867733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,4095,0.09518399834632874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,4095,0.09316266576449077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,8191,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,8191,0.020938667158285778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,8191,0.17935466766357422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,8191,0.17084266742070517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,16383,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,16383,0.02130666623512904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,16383,0.3535786469777425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,16383,0.3269493381182353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,float16,32767,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,1,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,128,1,float16,fp8,32767,0.02109866589307785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,1,0.013434667140245438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,1,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,1,0.014896000425020853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,3,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,3,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,float16,32767,0.9407893021901449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,3,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,64,0,1,float16,fp8,32767,0.6835626761118571
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,3,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,7,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,7,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,7,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,7,0.015370666980743408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,15,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,15,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,15,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,15,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,31,0.014671999961137772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,31,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,31,0.0174346665541331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,31,0.01691199963291486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,63,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,63,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,63,0.021002667645613354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,63,0.021104000508785248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,127,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,127,0.014645333091417948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,127,0.021189334491888683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,127,0.020970667401949566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,255,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,255,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,255,0.020879998803138733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,255,0.02093333254257838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,511,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,511,0.02143466720978419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,511,0.021370666722456615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,511,0.02316266546646754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,1023,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,1023,0.031632001201311745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,1023,0.021253332495689392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,1023,0.03352533280849457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,2047,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,2047,0.05395199855168661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,2047,0.021157334248224895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,2047,0.05397333204746246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,4095,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,4095,0.09512000282605489
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,4095,0.021087999145189922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,4095,0.09194133679072063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,8191,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,8191,0.1800480087598165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,8191,0.01926933353145917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,8191,0.1706506609916687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,16383,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,16383,0.02093333254257838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,16383,0.3535786469777425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,16383,0.3269546627998352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,float16,32767,0.015669333438078564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,128,1,float16,fp8,32767,0.021216000119845074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,1,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,float16,32767,0.9619519710540771
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,1,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,1,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,3,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,3,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,64,0,1,float16,fp8,32767,0.6832799911499023
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,3,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,3,0.01181866725285848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,7,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,7,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,7,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,7,0.012506666282812754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,15,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,15,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,15,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,15,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,31,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,31,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,63,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,63,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,127,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,127,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,127,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,127,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,255,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,255,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,255,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,255,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,511,0.012560000022252401
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,511,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,511,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,1023,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,511,0.012240000069141388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,1023,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,1023,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,1023,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,2047,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,2047,0.013823999712864557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,2047,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,2047,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,4095,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,4095,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,4095,0.01718933383623759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,4095,0.017690667261679966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,8191,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,8191,0.01937599976857503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,8191,0.01339200014869372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,8191,0.01951466624935468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,16383,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,16383,0.02550400048494339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,16383,0.013429333766301474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,16383,0.02479466547568639
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,float16,32767,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,float16,32767,0.04188799858093262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,128,1,float16,fp8,32767,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,1,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,1,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,64,0,1,float16,fp8,32767,0.03581333408753077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,1,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,1,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,3,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,3,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,3,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,7,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,7,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,7,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,7,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,15,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,15,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,15,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,15,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,31,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,31,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,31,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,31,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,63,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,63,0.010608000059922537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,63,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,63,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,127,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,127,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,127,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,255,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,127,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,255,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,255,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,511,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,511,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,511,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,1023,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,511,0.012485332787036896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,1023,0.013530666629473368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,1023,0.012144000579913458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,1023,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,2047,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,2047,0.016821333517630894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,2047,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,2047,0.016943999876578648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,4095,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,4095,0.019226666539907455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,4095,0.018826667219400406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,4095,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,8191,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,8191,0.02330133318901062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,8191,0.013530666629473368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,8191,0.02317333221435547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,16383,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,16383,0.038149334490299225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,16383,0.013429333766301474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,16383,0.03133333226044973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,float16,32767,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,128,1,float16,fp8,32767,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,float16,32767,0.05957333246866862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,64,0,1,float16,fp8,32767,0.05251200000445048
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,1,0.6584959824879965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,1,0.6590933402379354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,1,0.527562657992045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,1,0.5272106726964315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,3,0.6598933140436808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,3,0.5273866653442383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,3,0.6589493354161581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,3,0.5272853374481201
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,7,0.6813120047251383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,7,0.681114673614502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,7,0.5514666636784872
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,7,0.5504000186920166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,15,0.6953226725260416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,15,0.6950026353200277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,15,0.5747893253962199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,15,0.5743946631749471
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,31,0.8430559635162354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,31,0.8426559766133627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,31,0.7343893051147461
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,31,0.7342720031738281
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,63,0.8516960144042969
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,63,0.8516799608866373
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,63,0.7421600023905436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,63,0.7408426602681478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,float16,127,0.8572533130645752
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,float16,127,0.8580373128255209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,128,1,float16,fp8,127,0.7459253470102946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,64,0,1,float16,fp8,127,0.7451146443684896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,1,0.6585866610209147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,1,0.6599306662877401
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,1,0.5307626724243164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,1,0.5302346547444662
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,3,0.6590613524119059
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,3,0.530778686205546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,3,0.6589013338088989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,3,0.5301599899927775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,7,0.6805919806162516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,7,0.5517333348592123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,7,0.6828266779581705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,7,0.5519253412882487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,15,0.6932640075683594
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,15,0.5762133200963339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,15,0.6932266553243002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,15,0.5751680135726929
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,31,0.8434933026631674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,31,0.7350560029347738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,31,0.8445173104604086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,31,0.7352586587270101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,63,0.8522666295369467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,63,0.8517653147379557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,63,0.7425066630045573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,63,0.7420000235239664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,float16,127,0.8581013679504395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,float16,127,0.8581279913584391
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,128,1,float16,fp8,127,0.746064027150472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,64,0,1,float16,fp8,127,0.7455306847890218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,1,0.050144001841545105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,1,0.050517335534095764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,1,0.05053333441416422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,1,0.051183998584747314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,3,0.05028266708056132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,3,0.05021866659323374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,3,0.05055999755859375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,7,0.049829334020614624
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,3,0.05207466582457224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,7,0.05036800106366476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,7,0.05064000189304352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,7,0.050698667764663696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,15,0.050000001986821495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,15,0.0505973349014918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,15,0.05030400057633718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,15,0.050016000866889954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,31,0.050581331054369606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,31,0.050144001841545105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,31,0.05091733237107595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,31,0.05016533533732096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,63,0.05030933519204458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,63,0.05031466484069824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,63,0.051370665431022644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,63,0.05053333441416422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,float16,127,0.05172266562779745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,float16,127,0.05045333504676819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,128,1,float16,fp8,127,0.05222400029500326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,64,0,1,float16,fp8,127,0.05036800106366476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,1,0.07603733241558075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,1,0.07260799904664357
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,1,0.07503999769687653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,1,0.07243200143178304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,3,0.0751146674156189
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,3,0.07664533456166585
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,3,0.07234666744867961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,3,0.07247999807198842
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,7,0.07580799857775371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,7,0.07675733168919881
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,7,0.07254933317502339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,7,0.07243200143178304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,15,0.075573335091273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,15,0.07653333246707916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,15,0.07241599758466084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,15,0.0727040022611618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,31,0.07629333436489105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,31,0.07656533519426982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,31,0.07261333366235097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,31,0.07281599938869476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,63,0.07593599955240886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,63,0.07563733557860057
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,63,0.07285333176453908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,63,0.07237333556016286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,float16,127,0.07654933134714763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,float16,127,0.0765119989713033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,128,1,float16,fp8,127,0.07451733450094859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,64,0,1,float16,fp8,127,0.07431999842325847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,1,1.0446879863739014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,1,1.309007962544759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,1,1.3100319703420003
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,1,1.045423984527588
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,3,1.0451573530832927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,3,1.3094027042388916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,3,1.045733372370402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,3,1.3102986812591553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,7,1.0951840082804363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,7,1.352186679840088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,7,1.095040003458659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,7,1.3522987365722656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,15,1.1424533526102703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,15,1.1420106887817383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,15,1.380309263865153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,15,1.3816533088684082
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,31,1.6785227457682292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,31,1.6771893501281738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,31,1.4602400461832683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,31,1.4617973963419597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,fp8,63,1.474671999613444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,128,1,float16,float16,63,1.6954719225565593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,fp8,63,1.4764213562011719
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,64,0,1,float16,float16,63,1.6951146125793457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,1,1.0533173084259033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,1,1.3086826801300049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,1,1.0519839922587078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,1,1.3087093035380046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,3,1.0527413686116536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,3,1.052565336227417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,3,1.3079573313395183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,3,1.308282693227132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,7,1.0950667063395183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,7,1.3550400733947754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,7,1.3546826044718425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,7,1.095306634902954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,15,1.3793919881184895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,15,1.379701296488444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,15,1.1444640159606934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,15,1.144426663716634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,31,1.67903470993042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,31,1.680181344350179
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,31,1.462490717569987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,31,1.462949275970459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,float16,63,1.7021600405375164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,1,0.09122133255004883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,128,1,float16,fp8,63,1.4769226710001628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,float16,63,1.7118879954020183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,64,0,1,float16,fp8,63,1.4762132962544758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,1,0.09147733449935913
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,1,0.09261332949002583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,1,0.09176533420880635
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,3,0.09140266974767049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,3,0.09281599521636963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,3,0.0923466682434082
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,7,0.0918986697991689
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,7,0.09269866347312927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,7,0.0918346643447876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,7,0.0918239951133728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,15,0.09244267145792644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,15,0.09283733367919922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,15,0.09197333455085754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,3,0.09346133470535278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,31,0.09245333075523376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,31,0.09211200475692749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,31,0.09153067072232564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,31,0.09288000067075093
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,float16,63,0.09153067072232564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,float16,63,0.09126933415730794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,128,1,float16,fp8,63,0.09313600262006123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,1,0.14060266812642416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,1,0.1405119995276133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,15,0.09299733241399129
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,1,0.13742400209108988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,1,0.136543999115626
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,3,0.14138133327166238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,3,0.14127999544143677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,3,0.13753599921862283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,3,0.1373973290125529
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,7,0.14193066954612732
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,7,0.14199466506640115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,64,0,1,float16,fp8,63,0.09320533275604248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,7,0.13618666927019754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,7,0.13867732882499695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,15,0.14198399583498636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,15,0.13646933436393738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,15,0.14123732844988504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,15,0.13847999771436056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,31,0.1430293321609497
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,31,0.1405119995276133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,31,0.13793599605560303
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,31,0.13869333267211914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,float16,63,0.14247467120488486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,1,0.01964266722400983
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,1,0.018976000448067982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,float16,63,0.1405280033747355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,128,1,float16,fp8,63,0.13753066460291544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,1,0.01578666642308235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,1,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,3,0.0189280000825723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,3,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,3,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,3,0.014896000425020853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,7,0.019018666197856266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,64,0,1,float16,fp8,63,0.13782399892807007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,7,0.019386666516462963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,7,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,7,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,15,0.019274666905403137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,15,0.01952533299724261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,15,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,15,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,31,0.019386666516462963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,31,0.02089600016673406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,31,0.017237332959969837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,31,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,63,0.021040000021457672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,63,0.020917333662509918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,63,0.017344000438849132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,63,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,127,0.021130666136741638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,127,0.021274665991465252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,127,0.01834133391578992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,127,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,255,0.021130666136741638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,255,0.018751999984184902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,255,0.02102400114138921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,255,0.02346666653951009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,511,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,511,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,511,0.03150933235883713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,511,0.03150933235883713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,1023,0.020970667401949566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,1023,0.018383999665578205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,1023,0.0496319979429245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,1023,0.04805333415667216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,2047,0.021157334248224895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,2047,0.08467732866605122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,2047,0.017797333498795826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,2047,0.08293333152929942
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,4095,0.02102400114138921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,4095,0.15638933579126993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,4095,0.018581333259741466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,4095,0.15446933110555014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,8191,0.021066665649414062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,8191,0.01863466699918111
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,8191,0.2982613245646159
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,8191,0.29772265752156574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,float16,16383,0.020992000897725422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,128,1,float16,fp8,16383,0.018800000349680584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,float16,16383,0.6168800195058187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,1,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,1,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,1,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,1,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,3,0.01913600042462349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,64,0,1,float16,fp8,16383,0.6249920129776001
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,3,0.018826667219400406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,3,0.015599999576807022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,3,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,7,0.018778666853904724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,7,0.019359999646743137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,7,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,7,0.01533866673707962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,15,0.019205333044131596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,15,0.018981333822011948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,15,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,15,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,31,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,31,0.021082667013009388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,31,0.01911466692884763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,31,0.01718933383623759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,63,0.02128533273935318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,63,0.02065066620707512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,63,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,63,0.01897066707412402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,127,0.020949333906173706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,127,0.019679999599854153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,127,0.018735999862353008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,127,0.0185759998857975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,255,0.020341333001852036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,255,0.023525332411130268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,255,0.018085333208243053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,255,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,511,0.021136000752449036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,511,0.03198933353026708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,511,0.0170666662355264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,511,0.031045332551002502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,1023,0.02092266579469045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,1023,0.04996266464392344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,1023,0.020666666328907013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,1023,0.048026666045188904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,2047,0.02054399996995926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,2047,0.08500799536705017
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,2047,0.018346666047970455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,2047,0.0827946662902832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,4095,0.021136000752449036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,4095,0.15651733676592508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,4095,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,4095,0.15494400262832642
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,8191,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,8191,0.29871465762456256
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,8191,0.020026666422684986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,8191,0.29798932870229083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,float16,16383,0.021162666380405426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,1,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,1,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,128,1,float16,fp8,16383,0.01971199984351794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,1,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,float16,16383,0.6490240097045898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,3,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,1,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,3,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,3,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,64,0,1,float16,fp8,16383,0.6251253286997477
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,3,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,7,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,7,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,7,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,15,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,31,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,15,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,31,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,31,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,63,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,63,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,63,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,31,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,63,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,127,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,127,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,127,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,255,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,255,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,255,0.012378666549921036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,255,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,511,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,511,0.012634667257467905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,511,0.012736000120639801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,511,0.013557333499193192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,1023,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,1023,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,1023,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,1023,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,2047,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,2047,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,2047,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,2047,0.016997333616018295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,4095,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,4095,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,4095,0.01926400015751521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,4095,0.019466667125622433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,8191,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,8191,0.02700799951950709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,8191,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,8191,0.025600001215934753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,float16,16383,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,float16,16383,0.04162666698296865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,1,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,128,1,float16,fp8,16383,0.013541333377361298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,1,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,64,0,1,float16,fp8,16383,0.03583999971548716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,1,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,3,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,3,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,3,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,7,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,7,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,7,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,15,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,15,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,15,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,15,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,31,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,31,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,31,0.011663999408483505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,31,0.012373333175977072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,63,0.012549333274364471
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,127,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,63,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,127,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,127,0.011909333368142446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,255,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,255,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,255,0.011999999483426413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,255,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,511,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,511,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,511,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,1023,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,1023,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,1023,0.012698666503032049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,1023,0.013541333377361298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,2047,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,2047,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,2047,0.01894933357834816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,2047,0.01912533367673556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,4095,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,4095,0.023408000667889912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,4095,0.014746667196353277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,4095,0.021488000949223835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,8191,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,8191,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,8191,0.038986665507157646
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,8191,0.03143466760714849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,float16,16383,0.013701333353916803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,float16,16383,0.06001600126425425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,1,0.027306665976842243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,128,1,float16,fp8,16383,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,1,0.025093334416548412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,64,0,1,float16,fp8,16383,0.05179200073083242
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,3,0.027456000447273254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,7,0.027471999327341717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,3,0.025765334566434223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,7,0.025472000241279602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,15,0.032085334261258446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,15,0.029813334345817566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,31,0.029552000264326733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,31,0.03197333216667175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,63,0.033226666351159416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,63,0.02956266701221466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,127,0.037861332297325134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,127,0.035887998839219414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,255,0.05615466833114624
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,255,0.052095999320348106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,511,0.08914132912953694
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,511,0.08726400136947632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,1023,0.15876266360282898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,1023,0.15460800131162009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,2047,0.2977919975916545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,2047,0.2895946701367696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,4095,0.5747573375701904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,4095,0.5570400158564249
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,1,0.02769600103298823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,1,0.024485332270463307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,3,0.027589333554108936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,float16,8191,1.1397546927134197
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,3,0.025125332176685333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,7,0.027600000301996868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,7,0.025285333395004272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,1,128,0,1,float16,fp8,8191,1.0974986553192139
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,15,0.033557333052158356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,15,0.029546665648619335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,31,0.03145600110292435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,31,0.029765332738558452
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,63,0.03307733436425527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,63,0.029631999631722767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,127,0.038047999143600464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,127,0.035546667873859406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,255,0.0547680010398229
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,255,0.05217599868774414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,511,0.08897067109743755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,511,0.08679466446240743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,1023,0.1590986649195353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,1023,0.15412799517313638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,2047,0.2974720001220703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,2047,0.28775999943415326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,4095,0.5830560127894083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,4095,0.5559946695963541
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,1,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,1,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,3,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,float16,8191,1.1603466669718425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,3,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,7,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,7,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,2,128,0,1,float16,fp8,8191,1.1145280202229817
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,15,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,15,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,31,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,63,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,127,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,127,0.012714666624863943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,255,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,63,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,255,0.01239466667175293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,511,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,511,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,1023,0.014837333311637243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,1023,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,2047,0.02143466720978419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,2047,0.0194560003777345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,4095,0.039359999199708305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,4095,0.026314665873845417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,1,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,float16,8191,0.0598826656738917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,4,128,0,1,float16,fp8,8191,0.04196266829967499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,3,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,1,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,3,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,7,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,7,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,15,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,15,0.01349866638580958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,31,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,31,0.013349333157142004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,63,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,127,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,63,0.0136266661187013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,127,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,255,0.012650666137536367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,255,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,511,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,511,0.015333333363135656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,1023,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,1023,0.016949333250522614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,2047,0.0391146664818128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,2047,0.025807999074459076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,4095,0.059877331058184304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,4095,0.04031466692686081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,1,0.009237333511312803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,float16,8191,0.1018293301264445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,3,0.009301333377758661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,64,8,128,0,1,float16,fp8,8191,0.059434667229652405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,7,0.009183999771873156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,15,0.009077333534757296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,31,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,31,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,63,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,63,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,127,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,127,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,255,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,255,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,511,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,511,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,1023,0.0252960001428922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,1023,0.02939733366171519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,2047,0.033610666791598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,2047,0.03576533248027166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,4095,0.04964800179004669
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,4095,0.047872001926104225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,8191,0.0805866668621699
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,8191,0.0746559997399648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,16383,0.14215999841690063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,16383,0.12582932909329733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,1,0.009242666885256767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,1,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,float16,32767,0.26412800947825116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,3,0.008965333302815756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,7,0.00919999989370505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,1,128,0,1,float16,fp8,32767,0.22782933712005615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,15,0.00927466650803884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,31,0.009461333354314169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,63,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,63,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,127,0.010687999427318573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,127,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,255,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,255,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,511,0.02120000123977661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,511,0.01700266698996226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,1023,0.025818665822347004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,1023,0.029487999776999157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,2047,0.03349866718053818
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,2047,0.035589332381884255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,4095,0.049770668148994446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,4095,0.04770133395989736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,8191,0.08052800099054973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,8191,0.07431999842325847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,16383,0.14219733079274496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,16383,0.12587199608484903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,1,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,1,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,float16,32767,0.26522666215896606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,3,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,3,0.011786667009194693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,2,128,0,1,float16,fp8,32767,0.22773333390553793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,31,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,31,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,63,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,63,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,127,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,127,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,255,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,255,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,511,0.012719999998807907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,511,0.012661332885424295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,1023,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,1023,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,2047,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,2047,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,4095,0.014778666198253632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,4095,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,8191,0.019215999792019527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,8191,0.017488000293572743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,16383,0.021354667842388153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,16383,0.02046400060256322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,1,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,float16,32767,0.02229333420594533
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,3,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,3,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,4,128,0,1,float16,fp8,32767,0.02160000056028366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,7,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,7,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,15,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,15,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,31,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,63,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,127,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,63,0.012096000214417776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,255,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,511,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,511,0.012037333101034164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,1023,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,1023,0.012725333372751871
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,2047,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,2047,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,4095,0.015429332852363586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,4095,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,8191,0.016965333372354507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,8191,0.01709866647919019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,16383,0.019280000279347103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,16383,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,1,0.009328000247478485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,float16,32767,0.02327466756105423
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,1,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,64,8,128,0,1,float16,fp8,32767,0.019023999571800232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,3,0.009583999713261923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,3,0.01055466632048289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,7,0.00903466654320558
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,15,0.00916800027092298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,15,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,63,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,63,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,127,0.010591999938090643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,127,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,255,0.02107200026512146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,255,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,511,0.023455999791622162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,511,0.027306665976842243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,1023,0.029498666524887085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,1023,0.03145066648721695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,2047,0.042090664307276406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,2047,0.04196799794832865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,4095,0.06451733410358429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,4095,0.06065600117047628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,8191,0.11125866572062175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,8191,0.0995360016822815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,16383,0.20379199584325156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,16383,0.17675199111302695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,1,0.009375999992092451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,3,0.009109333157539368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,float16,32767,0.38791998227437335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,7,0.009039999917149544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,1,128,0,1,float16,fp8,32767,0.32873066266377765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,7,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,15,0.00919999989370505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,31,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,31,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,63,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,127,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,127,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,255,0.021317332983016968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,255,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,511,0.02422933280467987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,511,0.027376001079877216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,1023,0.029605334003766377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,1023,0.031770666440327965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,2047,0.042133331298828125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,2047,0.04181333382924398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,4095,0.06433600187301636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,4095,0.0620959997177124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,8191,0.11131200194358826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,8191,0.09924266735712688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,16383,0.20372267564137778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,16383,0.17670400937398276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,1,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,1,0.01198400060335795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,3,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,float16,32767,0.38812267780303955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,7,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,3,0.011882666498422623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,2,128,0,1,float16,fp8,32767,0.3306666612625122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,7,0.012666666259368261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,15,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,15,0.012426666915416718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,31,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,31,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,127,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,127,0.012223999947309494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,255,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,63,0.013861333330472311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,511,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,511,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,1023,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,1023,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,2047,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,2047,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,4095,0.017551999539136887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,4095,0.01700266698996226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,8191,0.019066666563351948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,8191,0.019018666197856266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,16383,0.019445333629846573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,16383,0.021701333423455555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,1,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,fp8,32767,0.02109866589307785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,4,128,0,1,float16,float16,32767,0.02346133440732956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,1,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,3,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,3,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,7,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,15,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,15,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,31,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,63,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,127,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,127,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,255,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,255,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,511,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,511,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,1023,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,1023,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,2047,0.014906667172908783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,2047,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,4095,0.016783999900023144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,4095,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,8191,0.019445333629846573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,8191,0.017386666188637417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,16383,0.02309333284695943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,16383,0.019648000597953796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,float16,32767,0.037530665596326195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,1,0.04582933088143667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,64,8,128,0,1,float16,fp8,32767,0.02332799881696701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,1,0.039781334499518074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,3,0.04781333108743032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,3,0.041840001940727234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,7,0.04176533222198486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,7,0.04783466458320618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,15,0.057999998331069946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,15,0.0524479995171229
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,31,0.05811200042565664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,31,0.052058666944503784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,63,0.05840000013510386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,63,0.051781331499417625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,127,0.06842133402824402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,127,0.06421866516272227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,255,0.10087466239929199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,255,0.0950986643632253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,511,0.16665599743525186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,511,0.1588213344415029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,1023,0.29839466015497845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,1023,0.2877013285954793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,2047,0.5623146692911783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,2047,0.5416266520818075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,1,0.04608533283074697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,1,0.039642666776975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,float16,4095,1.1050346692403157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,3,0.047925333182017006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,3,0.04172799984614054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,1,128,0,1,float16,fp8,4095,1.051082690556844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,7,0.04886400202910105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,15,0.058378666639328
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,15,0.05201066533724467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,31,0.05825066566467285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,7,0.04367466767628988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,31,0.05208533505598704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,63,0.0581279993057251
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,63,0.052757332722345986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,127,0.06418133278687795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,127,0.06942399839560191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,255,0.10102933645248413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,255,0.09529067079226176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,511,0.16666666666666666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,511,0.16125333309173584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,1023,0.29814932743708294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,1023,0.2876586715380351
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,2047,0.5705600182215372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,2047,0.5428640047709147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,1,0.014725333700577417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,1,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,3,0.013429333766301474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,3,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,fp8,4095,1.0576266447703044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,2,128,0,1,float16,float16,4095,1.1173813343048096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,7,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,7,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,15,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,15,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,31,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,31,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,63,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,63,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,127,0.013557333499193192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,127,0.013594667116800943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,255,0.013349333157142004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,255,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,511,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,511,0.015605332950750986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,1023,0.01922133316596349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,1023,0.019146667172511418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,2047,0.04022933294375738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,2047,0.02739733209212621
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,float16,4095,0.060991997520128884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,1,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,4,128,0,1,float16,fp8,4095,0.044031997521718345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,1,0.01553600033124288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,3,0.015504000087579092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,3,0.015376000354687372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,7,0.015381333728631338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,7,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,15,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,15,0.01540800059835116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,31,0.01590399940808614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,31,0.015344000111023584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,63,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,63,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,127,0.015381333728631338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,127,0.015466666469971338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,255,0.01545599972208341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,255,0.015466666469971338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,511,0.01939733326435089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,511,0.017504000415404636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,1023,0.036133334040641785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,1023,0.023557332654794056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,2047,0.060309335589408875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,2047,0.040661332507928215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,float16,4095,0.10203199585278828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,1,0.013482666263977686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,64,8,128,0,1,float16,fp8,4095,0.06123200058937073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,1,0.010591999938090643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,3,0.013733333597580591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,3,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,7,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,7,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,15,0.014815999815861383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,15,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,31,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,31,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,63,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,63,0.015402667224407196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,127,0.01591466615597407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,127,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,255,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,255,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,511,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,511,0.023285334308942158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,1023,0.03356266766786575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,1023,0.02985599885384242
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,2047,0.05421866476535797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,2047,0.049866666396458946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,4095,0.09527466694513957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,4095,0.08955732981363933
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,8191,0.17896000544230142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,8191,0.16906134287516275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,16383,0.34329601128896076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,16383,0.32868800560633343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,1,0.013663999736309052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,1,0.01071999967098236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,3,0.01421333352724711
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,float16,32767,0.672981341679891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,7,0.014240000396966934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,15,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,1,128,0,1,float16,fp8,32767,0.7333866755167643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,15,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,31,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,63,0.01540800059835116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,31,0.013482666263977686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,63,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,127,0.015749332805474598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,127,0.013573333621025085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,255,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,255,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,511,0.023397333920001984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,511,0.019589333484570186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,1023,0.029482667644818623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,1023,0.03412266572316488
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,2047,0.056128000219662987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,2047,0.04981866478919983
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,4095,0.09526933232943217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,4095,0.08982400099436443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,8191,0.17920533816019693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,8191,0.168778657913208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,16383,0.34310932954152423
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,1,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,16383,0.32839999596277875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,1,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,3,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,3,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,7,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,float16,32767,0.6821226278940836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,2,128,0,1,float16,fp8,32767,0.7330773671468099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,15,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,31,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,63,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,31,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,127,0.01071999967098236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,63,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,255,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,255,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,511,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,511,0.012752000242471695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,1023,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,1023,0.012698666503032049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,2047,0.01691199963291486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,2047,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,4095,0.01929066702723503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,4095,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,8191,0.021290667355060577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,8191,0.021055998901526134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,16383,0.024645333488782246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,16383,0.023391999304294586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,1,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,float16,32767,0.04161600023508072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,1,0.011690666278203329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,3,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,4,128,0,1,float16,fp8,32767,0.0277813325325648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,3,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,7,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,15,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,31,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,31,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,63,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,127,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,63,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,127,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,255,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,511,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,511,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,1023,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,1023,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,2047,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,2047,0.015482666591803232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,4095,0.01800000046690305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,4095,0.015423999478419622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,8191,0.01945066700379054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,8191,0.022853332261244457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,16383,0.02359466751416524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,16383,0.03754133234421412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,float16,32767,0.060234665870666504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,1,0.08725333213806152
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,64,8,128,0,1,float16,fp8,32767,0.03777066618204117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,1,0.07435200115044911
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,3,0.08872532844543457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,3,0.0765173335870107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,7,0.09113066395123799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,7,0.0784800002972285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,15,0.10958932836850484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,15,0.09799466530481975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,31,0.10937066872914632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,31,0.09910933176676433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,63,0.09914132952690125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,63,0.1113866666952769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,127,0.1302186648050944
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,127,0.11980266372362773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,255,0.19538666804631552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,255,0.1832533280054728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,511,0.32451732953389484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,511,0.3102506597836812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,1023,0.5883946816126505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,1023,0.5620266596476237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,1,0.0869760016600291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,1,0.07458666463692983
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,fp8,2047,1.066816012064616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,1,128,0,1,float16,float16,2047,1.1240479946136475
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,3,0.08973333239555359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,3,0.07634666562080383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,7,0.09118400017420451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,15,0.10964266459147136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,7,0.07905066510041554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,15,0.0981333355108897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,31,0.10967999696731567
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,31,0.09910933176676433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,63,0.11204266548156738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,63,0.09921600421269734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,127,0.13005866607030234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,127,0.11972799897193909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,255,0.19592533508936563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,255,0.18320000171661377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,511,0.32445333401362103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,511,0.3100480039914449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,1023,0.5608906745910645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,1023,0.598309318224589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,1,0.017008000363906223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,1,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,3,0.01716800034046173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,3,0.017045332739750545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,fp8,2047,1.0844213167826335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,2,128,0,1,float16,float16,2047,1.135434627532959
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,7,0.017466666797796886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,7,0.01710933322707812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,15,0.01695999999841054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,15,0.016869333883126576
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,31,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,31,0.01703466723362605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,63,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,63,0.017173333714405697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,127,0.01720533271630605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,127,0.01757866640885671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,255,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,255,0.017375999440749485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,511,0.021514666577180225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,511,0.021168000996112823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,1023,0.03759466608365377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,1023,0.02565866708755493
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,float16,2047,0.06266133487224579
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,1,0.01940800001223882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,1,0.019333332777023315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,4,128,0,1,float16,fp8,2047,0.046122665206591286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,3,0.020138667275508244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,3,0.019152000546455383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,7,0.02161066730817159
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,15,0.01933866615096728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,7,0.01940800001223882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,15,0.019797333826621372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,31,0.019589333484570186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,31,0.01978133370478948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,63,0.019050666441520054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,63,0.021301334102948506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,127,0.021573332448800404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,127,0.019487999379634857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,255,0.0206133338312308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,255,0.01926400015751521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,511,0.03439466655254364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,511,0.023728000621000927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,1023,0.055061335364977516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,1023,0.035536001125971474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,float16,2047,0.09675199786822002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,1,0.16687999169031778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,64,8,128,0,1,float16,fp8,2047,0.05879466732343038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,1,0.1405280033747355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,3,0.17292267084121704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,3,0.14441600441932678
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,7,0.15247467160224915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,7,0.1754186749458313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,15,0.21329599618911743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,15,0.18959999084472656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,31,0.2130933403968811
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,31,0.18957332770029703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,63,0.2141759991645813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,63,0.191594660282135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,127,0.25271467367808026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,127,0.23248000939687094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,255,0.38259732723236084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,255,0.3596160014470418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,float16,511,0.6402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,1,0.16729599237442017
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,1,128,0,1,float16,fp8,511,0.6114506721496582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,1,0.14058666427930197
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,3,0.17313599586486816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,3,0.14562132954597473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,7,0.17546667655309042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,7,0.15239999691645303
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,15,0.2121653358141581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,15,0.18966400623321533
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,31,0.21412799755732217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,31,0.1893226703008016
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,63,0.21436800559361777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,63,0.19129067659378052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,127,0.2531200051307678
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,127,0.2314079999923706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,255,0.3594026565551758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,255,0.3821386496225993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,1,0.023578666150569916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,1,0.023775999744733173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,float16,511,0.6543519894282023
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,2,128,0,1,float16,fp8,511,0.6113706827163696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,3,0.023408000667889912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,3,0.023589332898457844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,7,0.02348266790310542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,7,0.023157333334287006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,15,0.023386667172114056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,15,0.02332266668478648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,31,0.023546665906906128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,31,0.023183998962243397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,63,0.023311999936898548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,63,0.023200000325838726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,127,0.02346666653951009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,127,0.023285334308942158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,255,0.02312533309062322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,255,0.023247999449570973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,float16,511,0.03797333439191183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,1,0.0305226668715477
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,4,128,0,1,float16,fp8,511,0.029296000798543293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,1,0.02890666574239731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,3,0.03148266673088074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,3,0.02804800122976303
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,7,0.030602666238943737
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,7,0.029535998900731403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,15,0.031744000812371574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,15,0.027434666951497395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,31,0.03137599925200144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,31,0.029472000896930695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,63,0.03148799886306127
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,63,0.029135999580224354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,127,0.031167998909950256
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,127,0.029365333418051403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,255,0.03737066686153412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,255,0.029391999046007793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,float16,511,0.05693866809209188
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,64,8,128,0,1,float16,fp8,511,0.04085333396991094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,1,0.3272053400675456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,1,0.27139200766881305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,3,0.3391626675923665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,3,0.2823573350906372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,7,0.3448266585667928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,7,0.2967626651128133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,15,0.4191093444824219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,15,0.3717813491821289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,31,0.41925867398579914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,31,0.3722720146179199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,63,0.42290667692820233
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,63,0.37522133191426593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,127,0.4989493290583293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,127,0.45548800627390545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,1,0.3269760012626648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,float16,255,0.7568480173746744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,1,128,0,1,float16,fp8,255,0.7096746762593588
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,1,0.27316800753275555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,3,0.28365333875020343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,3,0.3403093417485555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,7,0.345301349957784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,7,0.2961813410123189
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,15,0.41895465056101483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,15,0.3717546860376994
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,31,0.4191146691640218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,31,0.3734079996744792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,63,0.4224746624628703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,63,0.3757013479868571
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,127,0.49881601333618164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,127,0.4554613431294759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,1,0.037685332198937736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,1,0.035573333501815796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,float16,255,0.7836639881134033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,2,128,0,1,float16,fp8,255,0.7102666695912679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,3,0.03743999948104223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,3,0.035599999129772186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,7,0.0379573330283165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,7,0.03573333223660787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,15,0.03796799977620443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,15,0.03571200122435888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,31,0.03741333385308584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,31,0.03578133384386698
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,63,0.038021333515644073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,63,0.035274667044480644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,127,0.03774400055408478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,127,0.03596800069014231
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,float16,255,0.042021334171295166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,4,128,0,1,float16,fp8,255,0.03675200045108795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,1,0.05236266553401947
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,1,0.046122665206591286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,3,0.05414933462937673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,3,0.045978665351867676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,7,0.05203733344872793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,7,0.04674133161703745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,15,0.05286400020122528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,15,0.04717866579691569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,31,0.05226666728655497
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,31,0.046053335070610046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,63,0.05209066470464071
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,63,0.046623999873797096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,127,0.05387733379999796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,1,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,127,0.045978665351867676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,1,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,3,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,float16,255,0.06142933170000712
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,3,0.015423999478419622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,64,8,128,0,1,float16,fp8,255,0.05116266508897146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,7,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,7,0.015802666544914246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,15,0.01461333284775416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,15,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,31,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,63,0.014757333944241205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,31,0.01937599976857503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,63,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,127,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,127,0.020901332298914593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,255,0.021146667500336964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,255,0.023498666783173878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,511,0.03146133323510488
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,511,0.03355200091997782
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,1023,0.051914667089780174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,1023,0.05410666763782501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,2047,0.09513599673906963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,2047,0.09266666571299235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,4095,0.1785279909769694
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,4095,0.16928533713022867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,8191,0.34512531757354736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,8191,0.32329599062601727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,16383,0.6904266675313314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,16383,0.6308106581370035
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,1,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,1,0.015306666493415833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,3,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,3,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,7,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,7,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,15,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,fp8,32767,1.4497386614481609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,15,0.016906666258970898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,31,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,31,0.021130666136741638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,63,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,1,128,0,1,float16,float16,32767,1.8879040082295735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,63,0.020869334538777668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,127,0.015930666277805965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,127,0.01923199991385142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,255,0.021231998999913532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,255,0.0232640008131663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,511,0.03137599925200144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,511,0.03328000009059906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,1023,0.05207466582457224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,1023,0.05258133510748545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,2047,0.09486400087674458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,2047,0.09167466560999553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,4095,0.18023999532063803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,4095,0.16901334126790366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,8191,0.34985601902008057
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,8191,0.32266666491826373
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,16383,0.7100000381469727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,16383,0.6303626696268717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,1,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,1,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,3,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,3,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,7,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,7,0.012682666381200155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,15,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,15,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,31,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,fp8,32767,1.5922187169392903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,31,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,2,128,0,1,float16,float16,32767,2.46012274424235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,127,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,127,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,255,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,255,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,511,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,511,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,1023,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,1023,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,2047,0.01706133286158244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,4095,0.017290666699409485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,4095,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,2047,0.017557332913080852
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,8191,0.022261333962281544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,8191,0.021344001094500225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,16383,0.03962666789690653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,16383,0.02741333345572154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,1,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,1,0.011744000017642975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,3,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,float16,32767,0.06021333237489065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,4,128,0,1,float16,fp8,32767,0.042992000778516136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,3,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,7,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,7,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,15,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,31,0.011637333780527115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,15,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,31,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,63,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,63,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,127,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,127,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,255,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,255,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,511,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,511,0.013669333110253016
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,1023,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,1023,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,2047,0.017018667111794155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,2047,0.017344000438849132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,4095,0.019466667125622433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,4095,0.021007999777793884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,8191,0.03774400055408478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,8191,0.02515733242034912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,16383,0.03775466730197271
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,16383,0.059936001896858215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,float16,32767,0.09984532992045085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,64,8,128,0,1,float16,fp8,32767,0.058575997749964394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,1,0.5333226521809896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,1,0.6462399959564209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,3,0.67194135983785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,3,0.557423988978068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,7,0.5851786533991495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,7,0.683072010676066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,15,0.7357920010884603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,15,0.8306879997253418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,31,0.7379733721415201
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,31,0.8307147026062012
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,63,0.8382453123728434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,63,0.7418719927469889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,float16,127,0.9927413463592529
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,1,128,0,1,float16,fp8,127,0.9026986757914225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,1,0.6447946627934774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,1,0.5367946624755859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,3,0.6726880073547363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,3,0.5576479832331339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,7,0.6850773493448893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,7,0.5864373445510864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,15,0.8310773372650146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,15,0.7358026504516602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,31,0.8323573271433512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,63,0.8383893171946207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,63,0.7422293027242025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,31,0.7383519808451334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,1,0.06611733138561249
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,float16,127,1.0323200225830078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,1,0.0614879975716273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,2,128,0,1,float16,fp8,127,0.9045813083648682
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,3,0.06596800188223521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,3,0.06062399844328562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,7,0.06671466430028279
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,7,0.06217066446940104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,15,0.0666240006685257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,15,0.0610346645116806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,31,0.06633600095907848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,31,0.06237333516279856
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,63,0.06675733129183452
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,63,0.061343997716903687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,float16,127,0.06659733255704244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,4,128,0,1,float16,fp8,127,0.06439466774463654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,1,0.09548266728719075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,1,0.08470933636029561
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,3,0.09562666217486064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,3,0.0851093331972758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,7,0.09525332848230998
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,7,0.08539199829101562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,15,0.0950986643632253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,15,0.084714670976003
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,31,0.09527466694513957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,31,0.08533333738644917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,63,0.09518399834632874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,63,0.08480000495910645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,float16,127,0.09734400113423665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,64,8,128,0,1,float16,fp8,127,0.08732266227404277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,1,1.061088005701701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,1,1.2826080322265625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,3,1.3419466018676758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,3,1.1094026565551758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,7,1.165765364964803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,7,1.3702774047851562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,15,1.4665600458780925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,15,1.6630132993062336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,31,1.4802826245625813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,31,1.6750987370808919
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,fp8,63,1.5013279914855957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,1,128,0,1,float16,float16,63,1.7188746134440105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,1,1.0697279771169026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,1,1.2874666849772136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,3,1.3449494043986003
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,3,1.1196746826171875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,7,1.1757813294728596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,7,1.3871359825134277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,15,1.478885332743327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,15,1.6818186442057292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,31,1.4993279774983723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,31,1.7062187194824219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,fp8,63,1.5397334098815918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,2,128,0,1,float16,float16,63,1.7299359639485676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,1,0.11803733309110005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,1,0.11338667074839275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,3,0.11798933148384094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,3,0.1141813298066457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,7,0.11286399761835735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,7,0.11981333295504253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,15,0.11797333757082622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,15,0.11374400059382121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,31,0.11800000071525574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,31,0.11359999577204387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,float16,63,0.11910399794578552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,4,128,0,1,float16,fp8,63,0.11347732941309611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,1,0.1592693328857422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,1,0.17732266585032144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,3,0.17722133795420328
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,3,0.15944000085194907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,7,0.15934933225313822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,7,0.17715734243392944
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,15,0.17717333634694418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,15,0.15955733259518942
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,31,0.15865600109100342
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,31,0.1771519978841146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,1,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,1,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,3,0.017530667285124462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,3,0.015114666273196539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,7,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,7,0.01543466622630755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,15,0.01916266605257988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,15,0.017605333278576534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,float16,63,0.17769600947697958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,31,0.0194560003777345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,31,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,63,0.019424000134070713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,64,8,128,0,1,float16,fp8,63,0.1597653329372406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,63,0.017583999782800674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,127,0.02367466688156128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,127,0.021338666478792827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,255,0.029674666623274486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,255,0.03128000100453695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,511,0.04983466863632202
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,511,0.048154667019844055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,1023,0.08477866649627686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,1023,0.082805335521698
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,2047,0.15521066387494406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,2047,0.1527253290017446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,4095,0.29606399933497113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,4095,0.29262399673461914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,8191,0.5799786647160848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,8191,0.572213331858317
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,1,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,1,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,3,0.018298666924238205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,float16,16383,1.2898613611857097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,3,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,7,0.019381333142518997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,1,128,0,1,float16,fp8,16383,1.3693706194559734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,7,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,15,0.020031999796628952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,15,0.017658667018016178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,31,0.01929066702723503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,31,0.017504000415404636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,63,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,63,0.019333332777023315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,127,0.02386133372783661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,127,0.02146666745344798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,255,0.03161599983771642
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,255,0.030192000170548756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,511,0.04986133178075155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,511,0.04842666784922282
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,1023,0.08281066517035167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,1023,0.08517866333325703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,2047,0.15491200486818948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,2047,0.15260266264279684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,4095,0.2983520030975342
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,4095,0.29177600145339966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,8191,0.5879413286844889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,8191,0.5724053382873535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,1,0.01240533341964086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,1,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,3,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,3,0.01181866725285848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,7,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,7,0.012042666474978128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,15,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,15,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,31,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,float16,16383,1.4013226826985676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,63,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,31,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,127,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,127,0.013477332890033722
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,2,128,0,1,float16,fp8,16383,1.4011146227518718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,255,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,255,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,511,0.01358933374285698
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,511,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,1023,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,1023,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,2047,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,2047,0.01922133316596349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,4095,0.02295999974012375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,4095,0.020879998803138733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,8191,0.03950933367013931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,8191,0.027914665639400482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,1,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,float16,16383,0.06011199951171875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,4,128,0,1,float16,fp8,16383,0.04240000247955322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,1,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,3,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,3,0.012522666404644648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,7,0.012383999923865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,15,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,15,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,31,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,31,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,63,0.01259200026591619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,63,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,127,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,127,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,255,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,255,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,511,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,511,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,1023,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,1023,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,2047,0.02124800036350886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,2047,0.019050666441520054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,4095,0.03828799972931544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,4095,0.0236160010099411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,8191,0.05956799785296122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,8191,0.037845333417256675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,1,0.01893866683046023
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,float16,16383,0.10046933094660442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,1,0.018922666708628338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,64,8,128,0,1,float16,fp8,16383,0.05829866727193197
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,1,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,1,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,3,0.019007999449968338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,3,0.018805333723624546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,3,0.015765332927306492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,7,0.01893866683046023
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,3,0.016021333634853363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,7,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,7,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,7,0.015749332805474598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,15,0.019018666197856266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,15,0.01932799940307935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,15,0.015637333194414776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,15,0.015610666324694952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,31,0.020517333100239437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,31,0.020128000527620316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,31,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,31,0.019626667102177937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,63,0.020175999651352566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,63,0.02096533278624217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,63,0.018960000326236088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,127,0.021162666380405426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,63,0.017850667238235474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,127,0.021141332884629566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,127,0.018677332748969395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,127,0.018698666244745255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,255,0.021370666722456615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,255,0.023647998770078022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,255,0.01841066653529803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,255,0.02144533395767212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,511,0.021162666380405426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,511,0.032058666149775185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,511,0.01966399947802226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,511,0.029482667644818623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,1023,0.021205333371957142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,1023,0.01828266680240631
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,1023,0.050250664353370667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,1023,0.04820266862710317
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,2047,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,2047,0.08542399605115254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,2047,0.01874133323629697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,2047,0.08270933230717976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,4095,0.0210506667693456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,4095,0.018885333091020584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,4095,0.15683733423550925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,4095,0.1546933352947235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,8191,0.02088533341884613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,8191,0.018415999909241993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,8191,0.298527995745341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,8191,0.2977866729100545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,float16,16383,0.021114667256673176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,128,1,float16,fp8,16383,0.01877333347996076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,float16,16383,0.6329013506571451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,1,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,1,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,3,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,1,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,3,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,3,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,7,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,64,0,1,float16,fp8,16383,0.6258026758829752
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,3,0.012186666329701742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,7,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,7,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,15,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,7,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,15,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,15,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,15,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,31,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,31,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,31,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,31,0.012655999511480331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,63,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,63,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,63,0.012586666891972223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,63,0.011813333878914515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,127,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,127,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,127,0.011850666254758835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,127,0.012058666596810022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,255,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,255,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,255,0.012703999876976013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,255,0.012618667135636011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,511,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,511,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,511,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,511,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,1023,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,1023,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,1023,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,1023,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,2047,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,2047,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,2047,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,2047,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,4095,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,4095,0.02093333254257838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,4095,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,4095,0.01939733326435089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,8191,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,8191,0.02595199892918269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,8191,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,8191,0.025050667424996693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,float16,16383,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,float16,16383,0.04213866591453552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,128,1,float16,fp8,16383,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,1,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,1,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,64,0,1,float16,fp8,16383,0.0359946663180987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,3,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,3,0.012400000045696894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,7,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,7,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,15,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,15,0.011823999385039011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,15,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,31,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,31,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,63,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,63,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,63,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,63,0.012768000364303589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,127,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,127,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,127,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,255,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,127,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,255,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,255,0.012597333639860153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,511,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,511,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,511,0.012576000144084295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,511,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,1023,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,1023,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,1023,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,2047,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,2047,0.01887999971707662
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,2047,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,2047,0.018895999838908512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,4095,0.01340266689658165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,4095,0.023258666197458904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,4095,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,4095,0.02313599983851115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,8191,0.014885333677132925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,8191,0.03919466584920883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,8191,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,8191,0.03219733387231827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,float16,16383,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,float16,16383,0.058431997895240784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,128,1,float16,fp8,16383,0.014773332824309668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,1,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,1,0.0124746672809124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,1,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,64,0,1,float16,fp8,16383,0.052229334910710655
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,3,0.01180800050497055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,1,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,3,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,3,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,3,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,7,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,7,0.01192533348997434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,7,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,7,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,15,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,15,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,15,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,15,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,31,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,31,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,31,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,31,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,63,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,63,0.011727999895811081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,63,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,63,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,127,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,127,0.012586666891972223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,127,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,127,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,255,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,255,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,255,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,511,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,255,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,511,0.013455999394257864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,511,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,511,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,1023,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,1023,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,1023,0.016879999389251072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,1023,0.016879999389251072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,2047,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,2047,0.023333333432674408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,2047,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,2047,0.023445333043734234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,4095,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,4095,0.039642666776975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,4095,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,4095,0.03156800071398417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,8191,0.015114666273196539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,8191,0.06145599981149038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,8191,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,8191,0.052517334620157875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,float16,16383,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,float16,16383,0.10296533505121867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,128,1,float16,fp8,16383,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,1,0.009381333366036415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,1,0.009066666786869368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,1,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,3,0.009072000160813332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,3,0.009253333633144697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,64,0,1,float16,fp8,16383,0.08719999591509502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,3,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,7,0.009082666908701261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,7,0.009413333609700203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,7,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,7,0.010677333921194077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,15,0.009450666606426239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,15,0.009050666665037474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,31,0.009226666763424873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,15,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,31,0.009066666786869368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,31,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,63,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,63,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,63,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,127,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,127,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,127,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,255,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,127,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,255,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,255,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,255,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,511,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,511,0.014208000153303146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,511,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,511,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,1023,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,1023,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,1023,0.016794666647911072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,1023,0.01703466723362605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,2047,0.017173333714405697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,2047,0.02293866624434789
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,2047,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,2047,0.019215999792019527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,4095,0.018885333091020584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,4095,0.027327999472618103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,4095,0.02123733361562093
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,4095,0.029946667452653248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,8191,0.019007999449968338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,8191,0.03748800108830134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,8191,0.03944533318281174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,8191,0.022389332453409832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,16383,0.019029332945744198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,16383,0.054885332783063255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,16383,0.021402666966120403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,16383,0.05747200051943461
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,32767,0.019050666441520054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,32767,0.0913706620534261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,32767,0.021402666966120403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,32767,0.08594666918118794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,float16,65535,0.019050666441520054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,128,1,float16,fp8,65535,0.02128533273935318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,float16,65535,0.1651893357435862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,1,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,1,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,1,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,64,0,1,float16,fp8,65535,0.14654933412869772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,3,0.010581333190202713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,3,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,7,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,7,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,15,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,15,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,15,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,15,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,31,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,31,0.01071999967098236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,31,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,63,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,63,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,63,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,63,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,127,0.01227733368674914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,127,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,127,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,127,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,255,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,255,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,511,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,511,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,511,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,511,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,1023,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,1023,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,1023,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,1023,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,2047,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,2047,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,2047,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,2047,0.013690666606028875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,4095,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,4095,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,4095,0.011786667009194693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,4095,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,8191,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,8191,0.01720000058412552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,8191,0.016714667280515034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,8191,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,16383,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,16383,0.021162666380405426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,16383,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,16383,0.020703999946514767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,32767,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,32767,0.023285334308942158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,32767,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,32767,0.022085333863894146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,float16,65535,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,float16,65535,0.024330665667851765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,1,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,128,1,float16,fp8,65535,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,1,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,3,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,3,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,3,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,3,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,7,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,64,0,1,float16,fp8,65535,0.02479466547568639
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,7,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,15,0.010570666442314783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,15,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,31,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,31,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,31,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,63,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,63,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,63,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,127,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,127,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,255,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,127,0.012245333443085352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,255,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,255,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,255,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,511,0.012037333101034164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,511,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,511,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,511,0.012319999436537424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,1023,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,1023,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,1023,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,1023,0.011749333391586939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,2047,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,2047,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,2047,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,2047,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,4095,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,4095,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,4095,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,4095,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,8191,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,8191,0.016842667013406754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,8191,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,8191,0.01684800038735072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,16383,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,16383,0.018144000321626663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,16383,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,16383,0.019071999937295914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,32767,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,32767,0.019893333315849304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,32767,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,32767,0.01905599981546402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,float16,65535,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,float16,65535,0.02160533269246419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,1,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,128,1,float16,fp8,65535,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,1,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,1,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,1,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,3,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,3,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,64,0,1,float16,fp8,65535,0.02081599955757459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,7,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,7,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,15,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,15,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,15,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,31,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,31,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,31,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,63,0.01051733394463857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,63,0.01055466632048289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,63,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,63,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,127,0.010602666685978571
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,127,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,127,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,255,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,255,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,255,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,255,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,511,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,511,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,511,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,511,0.012752000242471695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,1023,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,1023,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,1023,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,1023,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,2047,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,2047,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,2047,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,2047,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,4095,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,4095,0.014794666320085526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,4095,0.014794666320085526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,4095,0.012133333832025528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,8191,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,8191,0.01703466723362605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,8191,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,8191,0.01701333373785019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,16383,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,16383,0.01752000053723653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,16383,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,16383,0.01748266691962878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,32767,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,32767,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,32767,0.019205333044131596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,32767,0.018858666221300762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,float16,65535,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,128,1,float16,fp8,65535,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,float16,65535,0.023503998915354412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,1,0.009008000294367472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,1,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,1,0.00898133342464765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,1,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,64,0,1,float16,fp8,65535,0.021840001145998638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,3,0.008997333546479544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,3,0.009050666665037474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,3,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,3,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,7,0.008896000062425932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,7,0.009237333511312803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,7,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,15,0.008943999807039896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,15,0.008976000050703684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,15,0.010527999450763067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,31,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,15,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,31,0.009098666409651438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,63,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,31,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,63,0.010538666198650995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,127,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,127,0.010608000059922537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,127,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,127,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,255,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,255,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,255,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,255,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,511,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,511,0.014501333236694336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,511,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,511,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,1023,0.019130667050679524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,1023,0.021322667598724365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,1023,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,1023,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,2047,0.01912533367673556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,2047,0.025573333104451496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,2047,0.023024000227451324
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,2047,0.029552000264326733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,4095,0.019039999693632126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,4095,0.03385066737731298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,4095,0.02306666721900304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,4095,0.03545066714286804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,8191,0.019744000087181728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,8191,0.050101334849993386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,8191,0.021669333179791767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,8191,0.05006400247414907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,16383,0.01960533360640208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,16383,0.07914133369922638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,16383,0.02292799949645996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,16383,0.07682666679223378
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,32767,0.019391999890406925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,32767,0.14086932937304178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,32767,0.02316266546646754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,32767,0.126202662785848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,float16,65535,0.019648000597953796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,float16,65535,0.2629599968592326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,128,1,float16,fp8,65535,0.02327466756105423
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,1,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,1,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,1,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,1,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,3,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,3,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,3,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,7,0.010538666198650995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,64,0,1,float16,fp8,65535,0.23030932744344076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,7,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,7,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,15,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,31,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,31,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,31,0.012250666817029318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,31,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,63,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,127,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,63,0.011994666109482447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,127,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,127,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,255,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,255,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,255,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,255,0.01231466606259346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,511,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,511,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,511,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,511,0.013621332744757334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,1023,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,1023,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,1023,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,2047,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,1023,0.012367999802033106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,2047,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,2047,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,4095,0.011658667276302973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,2047,0.013658666362365087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,4095,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,4095,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,4095,0.014069333672523499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,8191,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,8191,0.01701333373785019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,8191,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,8191,0.01695466662446658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,16383,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,16383,0.019205333044131596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,16383,0.011941333611806234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,16383,0.01950399950146675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,32767,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,32767,0.02107200026512146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,32767,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,32767,0.021141332884629566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,float16,65535,0.011962667107582092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,float16,65535,0.02329600105683009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,128,1,float16,fp8,65535,0.011866666376590729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,1,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,1,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,1,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,3,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,3,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,64,0,1,float16,fp8,65535,0.023370665808518726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,7,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,7,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,7,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,15,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,15,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,31,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,31,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,63,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,31,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,63,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,63,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,127,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,127,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,255,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,255,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,255,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,511,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,511,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,511,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,511,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,1023,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,1023,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,1023,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,1023,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,2047,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,2047,0.013514666507641474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,2047,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,2047,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,4095,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,4095,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,4095,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,4095,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,8191,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,8191,0.017397332936525345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,8191,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,8191,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,16383,0.012346666306257248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,16383,0.019152000546455383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,16383,0.01179733375708262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,16383,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,32767,0.01947733387351036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,32767,0.011653333902359009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,32767,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,32767,0.019152000546455383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,float16,65535,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,fp8,65535,0.02348800003528595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,128,1,float16,fp8,65535,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,64,0,1,float16,float16,65535,0.023391999304294586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,1,0.010586666564146677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,1,0.010575999816258749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,1,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,3,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,3,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,3,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,7,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,7,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,7,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,7,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,15,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,15,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,15,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,31,0.01062400018175443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,31,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,31,0.011727999895811081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,63,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,63,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,63,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,127,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,127,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,127,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,255,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,255,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,255,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,511,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,511,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,511,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,511,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,1023,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,1023,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,1023,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,1023,0.012442667037248611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,2047,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,2047,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,2047,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,2047,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,4095,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,4095,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,4095,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,4095,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,8191,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,8191,0.01720000058412552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,8191,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,8191,0.017125333348910015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,16383,0.011663999408483505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,16383,0.019130667050679524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,16383,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,16383,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,32767,0.0116799995303154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,32767,0.023141334454218548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,32767,0.012709333250919977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,32767,0.02199466774861018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,float16,65535,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,float16,65535,0.036544000109036766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,128,1,float16,fp8,65535,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,1,0.027488000690937042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,1,0.027130665878454845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,1,0.023541333774725597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,1,0.023397333920001984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,3,0.027776000400384266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,64,0,1,float16,fp8,65535,0.029605334003766377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,3,0.027306665976842243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,3,0.0232640008131663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,3,0.023242667317390442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,7,0.029098667204380035
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,7,0.02752533306678136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,7,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,7,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,15,0.02779199928045273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,15,0.029365333418051403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,15,0.02513066679239273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,15,0.025221332907676697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,31,0.03356799980004629
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,31,0.033386667569478355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,31,0.029520000020662945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,31,0.02935466667016347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,63,0.033530667424201965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,63,0.029578665892283123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,63,0.034490667283535004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,63,0.030832000076770782
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,127,0.0339626669883728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,127,0.033488000432650246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,127,0.031370667119820915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,255,0.03383466601371765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,255,0.039477333426475525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,127,0.03183466692765554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,255,0.031221332649389904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,255,0.035717333356539406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,511,0.03350399931271871
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,511,0.057002668579419456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,511,0.03150933235883713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,511,0.052149335543314614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,1023,0.033488000432650246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,1023,0.031114667654037476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,1023,0.08710400263468425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,1023,0.09161067008972168
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,2047,0.033439998825391136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,2047,0.029658667743206024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,2047,0.16040000319480896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,2047,0.15734933813412985
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,4095,0.033488000432650246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,4095,0.03141333411137263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,4095,0.2998613317807515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,float16,8191,0.034074666599432625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,4095,0.29627732435862225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,1,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,128,1,float16,fp8,8191,0.030058667063713074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,1,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,float16,8191,0.5870240132013956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,1,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,1,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,3,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,3,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,64,0,1,float16,fp8,8191,0.5725440184275309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,3,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,7,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,7,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,15,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,15,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,15,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,15,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,31,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,31,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,31,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,31,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,63,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,63,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,63,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,127,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,127,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,127,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,127,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,255,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,255,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,511,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,511,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,511,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,511,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,1023,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,1023,0.014773332824309668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,1023,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,1023,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,2047,0.015674666812022526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,2047,0.020581333587567013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,2047,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,2047,0.0195573332409064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,4095,0.014720000326633453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,4095,0.025797332326571148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,4095,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,4095,0.025445332129796345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,float16,8191,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,float16,8191,0.04370133578777313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,128,1,float16,fp8,8191,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,1,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,1,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,1,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,1,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,64,0,1,float16,fp8,8191,0.03596800069014231
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,3,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,3,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,3,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,3,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,7,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,7,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,7,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,7,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,15,0.012752000242471695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,15,0.012650666137536367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,15,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,31,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,15,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,31,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,31,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,31,0.013573333621025085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,63,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,63,0.012741333494583765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,63,0.013642666240533194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,127,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,63,0.01434133326013883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,127,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,127,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,127,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,255,0.012736000120639801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,255,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,255,0.013786666095256805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,255,0.012703999876976013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,511,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,511,0.014954666296641031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,511,0.014181333283583323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,511,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,1023,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,1023,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,1023,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,1023,0.015439999600251516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,2047,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,2047,0.025125332176685333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,2047,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,2047,0.02329600105683009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,4095,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,4095,0.03976000100374222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,4095,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,4095,0.03195200115442276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,float16,8191,0.01624533285697301
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,float16,8191,0.06018133461475372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,128,1,float16,fp8,8191,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,64,0,1,float16,fp8,8191,0.05306666592756907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,1,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,1,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,1,0.014469332993030548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,1,0.014752000570297241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,3,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,3,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,3,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,3,0.014831999937693277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,7,0.013482666263977686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,7,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,7,0.01543466622630755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,7,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,15,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,15,0.014885333677132925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,15,0.015306666493415833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,15,0.014906667172908783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,31,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,31,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,31,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,31,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,63,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,63,0.014074667046467463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,63,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,63,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,127,0.014106666048367819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,127,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,127,0.015642666568358738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,127,0.014682666709025701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,255,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,255,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,255,0.013594667116800943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,255,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,511,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,511,0.017349333812793095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,511,0.016976000120242436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,511,0.014815999815861383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,1023,0.014826666563749313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,1023,0.021509334444999695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,1023,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,1023,0.021359999974568684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,2047,0.017722666263580322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,2047,0.039664000272750854
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,2047,0.01714666684468587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,2047,0.03366400053103765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,4095,0.017674667139848072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,4095,0.06098133325576782
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,4095,0.017344000438849132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,4095,0.05287466446558634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,float16,8191,0.10337600111961365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,float16,8191,0.017338667064905167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,1,0.009359999870260557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,128,1,float16,fp8,8191,0.017792000124851864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,1,0.009152000149091085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,1,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,64,0,1,float16,fp8,8191,0.08815999825795491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,3,0.009306666751702627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,3,0.00927466650803884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,3,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,7,0.009258666386206945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,7,0.009088000282645226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,7,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,7,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,15,0.009375999992092451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,15,0.009061333412925402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,15,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,31,0.009066666786869368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,31,0.010527999450763067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,31,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,31,0.010687999427318573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,63,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,63,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,127,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,127,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,127,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,127,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,255,0.012074666718641916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,255,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,255,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,255,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,511,0.020015999674797058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,511,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,511,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,511,0.021365332106749218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,1023,0.02309866746266683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,1023,0.02345066765944163
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,1023,0.02739733209212621
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,1023,0.019808000574509304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,2047,0.019530666371186573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,2047,0.03025600065787633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,2047,0.025610665480295818
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,2047,0.03233066697915395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,4095,0.01924266666173935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,4095,0.04197333256403605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,4095,0.02532800038655599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,4095,0.04293866455554962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,8191,0.020682666450738907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,8191,0.06491200129191081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,8191,0.025125332176685333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,8191,0.06284266710281372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,16383,0.020282667130231857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,16383,0.11137066284815471
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,16383,0.025226667523384094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,16383,0.10171733299891154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,32767,0.02103466788927714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,32767,0.20402133464813232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,32767,0.02388266722361247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,32767,0.17726399501164755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,float16,65535,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,128,1,float16,fp8,65535,0.025087999800841015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,float16,65535,0.38737066586812335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,1,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,1,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,1,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,3,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,3,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,7,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,64,0,1,float16,fp8,65535,0.331002672513326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,7,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,7,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,7,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,15,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,15,0.01181866725285848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,15,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,31,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,31,0.012655999511480331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,31,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,63,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,63,0.011658667276302973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,127,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,127,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,127,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,127,0.011493333925803503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,255,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,255,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,255,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,255,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,511,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,511,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,511,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,511,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,1023,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,1023,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,1023,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,2047,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,2047,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,2047,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,2047,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,4095,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,4095,0.016842667013406754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,4095,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,4095,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,8191,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,8191,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,8191,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,8191,0.017504000415404636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,16383,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,16383,0.019343999524911244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,16383,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,16383,0.019109333554903667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,32767,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,32767,0.023418667415777843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,32767,0.021407999098300934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,32767,0.013381333400805792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,float16,65535,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,1,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,float16,65535,0.027295999228954315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,128,1,float16,fp8,65535,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,64,0,1,float16,fp8,65535,0.027327999472618103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,1,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,3,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,3,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,3,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,7,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,7,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,15,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,15,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,15,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,31,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,63,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,31,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,63,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,63,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,127,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,127,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,127,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,255,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,255,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,255,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,511,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,511,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,511,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,1023,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,1023,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,1023,0.011760000139474869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,1023,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,2047,0.013477332890033722
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,2047,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,2047,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,2047,0.015471999843915304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,4095,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,4095,0.01752000053723653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,4095,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,4095,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,8191,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,8191,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,8191,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,8191,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,16383,0.021173333128293354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,16383,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,16383,0.013434667140245438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,16383,0.02032533288002014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,32767,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,32767,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,32767,0.025279998779296875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,32767,0.023669332265853882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,float16,65535,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,float16,65535,0.03887466589609782
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,1,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,128,1,float16,fp8,65535,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,1,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,3,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,3,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,64,0,1,float16,fp8,65535,0.031770666440327965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,3,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,3,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,7,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,7,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,7,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,15,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,15,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,15,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,15,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,31,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,31,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,31,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,31,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,63,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,63,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,63,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,63,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,127,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,127,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,255,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,127,0.012047999848922094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,255,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,255,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,255,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,511,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,511,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,511,0.012240000069141388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,511,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,1023,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,1023,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,1023,0.012426666915416718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,1023,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,2047,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,2047,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,2047,0.013674666484196981
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,2047,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,4095,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,4095,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,4095,0.01532799998919169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,4095,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,8191,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,8191,0.0186666672428449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,8191,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,8191,0.017850667238235474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,16383,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,16383,0.023599999646345775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,16383,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,16383,0.02317333221435547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,32767,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,32767,0.03782399992148081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,32767,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,32767,0.031370667119820915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,float16,65535,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,float16,65535,0.0581226646900177
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,128,1,float16,fp8,65535,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,1,0.04831466575463613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,1,0.04804799954096476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,1,0.03976533313592275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,1,0.040522667268911995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,3,0.047797332207361855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,64,0,1,float16,fp8,65535,0.05188799897829691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,3,0.04880533119042715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,3,0.039749334255854286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,7,0.04833599925041199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,3,0.039503999054431915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,7,0.05014933149019877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,7,0.04158399999141693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,7,0.04178666571776072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,15,0.050000001986821495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,15,0.04178666571776072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,15,0.04966400067011515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,15,0.043237333496411644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,31,0.058335999647776283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,31,0.059818665186564125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,31,0.05233600238958994
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,31,0.052202666799227394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,63,0.059994667768478394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,63,0.05994666616121928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,63,0.05197866757710775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,63,0.051962668697039284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,127,0.060271998246510826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,127,0.05379733443260193
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,127,0.060309335589408875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,127,0.05386666456858317
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,255,0.06052800019582113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,255,0.06912533442179362
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,255,0.05272533496220907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,255,0.06302399933338165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,511,0.06018133461475372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,511,0.10313066840171814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,511,0.053370664517084755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,511,0.09548266728719075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,1023,0.06033066908518473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,1023,0.05188799897829691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,1023,0.1676106651624044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,1023,0.1609333356221517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,2047,0.0610453337430954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,2047,0.052485331892967224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,2047,0.30010666449864704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,2047,0.2937493324279785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,float16,4095,0.06085866689682007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,128,1,float16,fp8,4095,0.0521066685517629
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,float16,4095,0.5746133327484131
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,1,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,1,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,64,0,1,float16,fp8,4095,0.5572906732559204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,1,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,1,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,3,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,3,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,3,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,3,0.013647999614477158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,7,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,7,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,7,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,7,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,15,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,15,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,15,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,15,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,31,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,31,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,31,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,63,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,31,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,63,0.012752000242471695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,63,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,63,0.012741333494583765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,127,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,127,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,127,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,127,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,255,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,255,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,255,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,255,0.014181333283583323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,511,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,511,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,511,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,511,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,1023,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,1023,0.013610667238632837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,1023,0.017456000049908955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,1023,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,2047,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,2047,0.01884799947341283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,2047,0.027445333699385326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,2047,0.02550933261712392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,float16,4095,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,128,1,float16,fp8,4095,0.017125333348910015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,float16,4095,0.04397333165009817
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,1,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,1,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,64,0,1,float16,fp8,4095,0.035589332381884255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,1,0.015392000476519266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,1,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,3,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,3,0.014655999839305878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,3,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,3,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,7,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,7,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,7,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,7,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,15,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,15,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,15,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,15,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,31,0.01488000030318896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,31,0.014426667243242264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,31,0.014842666685581207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,31,0.015674666812022526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,63,0.014826666563749313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,63,0.015279999623696009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,63,0.014922666052977243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,63,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,127,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,127,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,127,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,127,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,255,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,255,0.013744000345468521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,255,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,255,0.014757333944241205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,511,0.015344000111023584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,511,0.016938666502634685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,511,0.014885333677132925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,511,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,1023,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,1023,0.021375998854637146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,1023,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,1023,0.021482666333516438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,2047,0.017488000293572743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,2047,0.040192000567913055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,2047,0.01757866640885671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,2047,0.033546666304270424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,float16,4095,0.01758933315674464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,float16,4095,0.06235733131567637
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,128,1,float16,fp8,4095,0.01754133279124896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,1,0.017349333812793095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,1,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,64,0,1,float16,fp8,4095,0.05398933092753092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,1,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,1,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,3,0.01720000058412552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,3,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,3,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,7,0.017024000485738117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,7,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,3,0.020080000162124634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,7,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,7,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,15,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,15,0.017504000415404636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,15,0.018005333840847015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,15,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,31,0.017263999829689663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,31,0.017360000560681026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,31,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,63,0.017173333714405697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,31,0.019167999426523846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,63,0.01732800031701724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,63,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,63,0.01732800031701724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,127,0.017808000246683758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,127,0.017925333231687546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,127,0.01718933383623759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,127,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,255,0.01916266605257988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,255,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,255,0.017616000026464462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,255,0.018053332964579265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,511,0.019199999670187633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,511,0.023141334454218548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,511,0.023242667317390442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,511,0.017717332889636356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,1023,0.01932799940307935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,1023,0.03519999980926514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,1023,0.018826667219400406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,1023,0.029696000119050343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,2047,0.021333334346612293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,2047,0.05740800003210703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,2047,0.021040000021457672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,2047,0.05198400219281515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,float16,4095,0.021344001094500225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,float16,4095,0.09484799702962239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,128,1,float16,fp8,4095,0.021136000752449036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,64,0,1,float16,fp8,4095,0.08163733283678691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,1,0.08932800094286601
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,1,0.08921600381533305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,1,0.0727040022611618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,1,0.07294400036334991
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,3,0.08898666501045227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,3,0.08901866277058919
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,3,0.07302933434645335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,3,0.0724480003118515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,7,0.09129599730173747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,7,0.09100799759229024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,7,0.07500266532103221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,7,0.07566399872303009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,15,0.09332799911499023
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,15,0.09315733114878337
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,15,0.07843733330567677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,15,0.07838933169841766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,31,0.11146133144696553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,31,0.11133333047231038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,31,0.09869866569836934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,31,0.09728533029556274
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,63,0.11343466242154439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,63,0.09886399904886882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,63,0.11339199542999268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,63,0.09918933113416036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,127,0.11357333262761433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,127,0.11342933773994446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,127,0.0990826686223348
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,127,0.0990613301595052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,255,0.11361066500345866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,255,0.13215999801953635
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,255,0.09913600484530131
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,255,0.11944533387819926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,511,0.11377066373825073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,511,0.09918933113416036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,511,0.19766400257746378
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,511,0.18344533443450928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,float16,1023,0.11353600025177002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,128,1,float16,fp8,1023,0.09907199939092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,float16,1023,0.3264960050582886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,1,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,1,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,64,0,1,float16,fp8,1023,0.314303994178772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,1,0.015402667224407196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,1,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,3,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,3,0.016229332735141117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,3,0.015418666104475657
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,3,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,7,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,7,0.015429332852363586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,7,0.015418666104475657
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,7,0.015685333559910457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,15,0.01534933348496755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,15,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,15,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,31,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,15,0.01562133307258288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,31,0.015509333461523056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,31,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,31,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,63,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,63,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,63,0.015696000307798386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,63,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,127,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,127,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,127,0.015594666202863058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,127,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,255,0.015392000476519266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,255,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,255,0.01590399940808614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,255,0.014954666296641031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,511,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,511,0.01907733331123988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,511,0.01926400015751521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,511,0.015487999965747198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,float16,1023,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,float16,1023,0.02573866645495097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,128,1,float16,fp8,1023,0.016389333953460056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,64,0,1,float16,fp8,1023,0.023354666928450268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,1,0.018768000106016796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,1,0.018837332725524902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,1,0.018960000326236088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,3,0.019280000279347103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,1,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,3,0.018901333212852478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,3,0.019002666076024372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,3,0.019199999670187633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,7,0.019093333433071773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,7,0.017727999637524288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,7,0.019120000302791595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,7,0.019215999792019527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,15,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,15,0.019071999937295914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,15,0.017685333887736004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,15,0.018901333212852478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,31,0.01754666616519292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,31,0.019199999670187633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,31,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,31,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,63,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,63,0.01889066646496455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,63,0.017194667210181553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,63,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,127,0.0191040001809597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,127,0.017759999881188076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,127,0.01894933357834816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,127,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,255,0.019205333044131596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,255,0.019365333020687103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,255,0.01903466631968816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,255,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,511,0.02334933231274287
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,511,0.01956266661485036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,511,0.01825599993268649
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,511,0.022645334402720135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,float16,1023,0.01941866676012675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,128,1,float16,fp8,1023,0.01923199991385142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,float16,1023,0.03438399980465571
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,64,0,1,float16,fp8,1023,0.031514666974544525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,1,0.025477332373460133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,1,0.02588266630967458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,1,0.025573333104451496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,1,0.025759999950726826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,3,0.025578667720158894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,3,0.025839999318122864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,3,0.025792000194390614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,3,0.025578667720158894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,7,0.025578667720158894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,7,0.025829332570234936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,7,0.025781333446502686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,7,0.025285333395004272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,15,0.025685332715511322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,15,0.025386666258176167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,15,0.025573333104451496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,15,0.02611200014750163
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,31,0.02537599951028824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,31,0.02624000112215678
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,31,0.02555199960867564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,31,0.02521066615978877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,63,0.025594666600227356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,63,0.025349333882331848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,63,0.025306666890780132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,63,0.025546667476495106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,127,0.025258667767047882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,127,0.025093334416548412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,127,0.02526933451493581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,127,0.0259253333012263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,255,0.025370667378107708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,255,0.02518400053183238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,255,0.0252960001428922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,255,0.025786665578683216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,511,0.027327999472618103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,511,0.0374293327331543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,511,0.025418666501839954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,511,0.03379733363787333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,float16,1023,0.02735466758410136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,float16,1023,0.05596266686916351
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,128,1,float16,fp8,1023,0.02566933383544286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,64,0,1,float16,fp8,1023,0.050741334756215416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,1,0.17082667350769043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,1,0.17098132769266763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,1,0.1385546624660492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,1,0.13929599523544312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,3,0.17083199818929037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,3,0.17069866259892783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,3,0.13825066884358725
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,3,0.13878400127092996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,7,0.17534399032592773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,7,0.17638933658599854
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,7,0.14412800470987955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,7,0.144405335187912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,15,0.179258664449056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,15,0.17911465962727866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,15,0.14998400211334229
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,15,0.15014933546384177
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,31,0.18937599658966064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,31,0.21708800395329794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,31,0.21631999810536703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,31,0.1896160046259562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,63,0.21795733769734701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,63,0.21804267168045044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,63,0.1911946733792623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,63,0.19131733973821005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,127,0.19148266315460205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,127,0.21987199783325195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,127,0.2197493314743042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,127,0.19174933433532715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,255,0.19167999426523843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,255,0.21988266706466675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,255,0.2569173375765483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,255,0.23041067520777384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,fp8,511,0.19123733043670654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,128,1,float16,float16,511,0.22040534019470215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,float16,511,0.38610132535298664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,1,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,1,0.020394666741291683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,64,0,1,float16,fp8,511,0.3612639904022217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,1,0.021066665649414062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,1,0.021221332252025604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,3,0.020288000504175823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,3,0.01937066639463107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,3,0.02128533273935318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,3,0.021381333470344543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,7,0.019461333751678467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,7,0.020527999848127365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,7,0.01945066700379054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,7,0.021168000996112823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,15,0.020842666427294414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,15,0.019658666104078293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,15,0.019989332805077236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,15,0.021909333765506744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,31,0.019205333044131596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,31,0.019381333142518997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,31,0.02139200021823247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,31,0.021226666867733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,63,0.019717333217461903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,63,0.01964266722400983
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,63,0.02120000123977661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,63,0.02103466788927714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,127,0.019440000255902607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,127,0.019199999670187633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,127,0.021226666867733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,127,0.019695999721686046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,255,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,255,0.019632000476121902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,255,0.021327999730904896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,255,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,float16,511,0.021221332252025604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,float16,511,0.025455998877684276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,128,1,float16,fp8,511,0.02139200021823247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,1,0.025786665578683216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,1,0.027119999130566914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,64,0,1,float16,fp8,511,0.026208000878492992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,1,0.025407999753952026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,1,0.025557334224383037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,3,0.02610666553179423
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,3,0.026709333062171936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,3,0.025744001070658367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,3,0.025445332129796345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,7,0.025605333348115284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,7,0.02622933437426885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,7,0.025968000292778015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,7,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,15,0.025402667621771496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,15,0.025568000972270966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,15,0.025392000873883564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,15,0.026181332767009735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,31,0.0258240004380544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,31,0.027727998793125153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,31,0.025253333151340485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,63,0.026352000733216602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,31,0.025568000972270966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,63,0.027445333699385326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,63,0.02603200078010559
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,63,0.025765334566434223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,127,0.025498665869235992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,127,0.026005332668622334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,127,0.025248001019159954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,127,0.026186667382717133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,255,0.027621333797772724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,255,0.025706666211287182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,255,0.025455998877684276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,255,0.0262719988822937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,float16,511,0.027589333554108936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,float16,511,0.03788800040880839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,128,1,float16,fp8,511,0.025333332518736523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,64,0,1,float16,fp8,511,0.03538133452335993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,1,0.03992533435424169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,1,0.03991466760635376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,1,0.03941333293914795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,1,0.040005333721637726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,3,0.03948266555865606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,3,0.0399893323580424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,3,0.039503999054431915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,3,0.04037333279848099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,7,0.039647998909155525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,7,0.040234667559464775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,7,0.039861333866914116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,7,0.04019733270009359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,15,0.04029866556326548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,15,0.0401706670721372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,15,0.03946666667858759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,15,0.039546666045983635
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,31,0.04030933231115341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,31,0.04011733333269755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,31,0.04013866682847341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,31,0.03994133323431015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,63,0.040250666439533234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,63,0.040021332601706185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,63,0.03994133323431015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,63,0.039461334546407066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,127,0.03978666663169861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,127,0.03987200061480204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,127,0.03942399968703588
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,127,0.03953066716591517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,255,0.04172799984614054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,255,0.042768001556396484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,255,0.03967999915281931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,255,0.03982399900754293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,float16,511,0.04173333446184794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,float16,511,0.061466669042905174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,128,1,float16,fp8,511,0.03972266614437103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,1,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,1,0.01482133318980535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,64,0,1,float16,fp8,511,0.05816533168156942
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,1,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,1,0.01051733394463857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,3,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,3,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,3,0.014485333114862442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,3,0.00949866697192192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,7,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,7,0.014405333747466406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,7,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,7,0.01051733394463857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,15,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,15,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,15,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,31,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,31,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,31,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,63,0.015477333217859268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,63,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,63,0.012736000120639801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,63,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,127,0.016165333489576977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,127,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,127,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,127,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,255,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,255,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,255,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,255,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,511,0.016976000120242436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,511,0.018965333700180054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,511,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,511,0.014655999839305878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,1023,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,1023,0.022965334355831146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,1023,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,1023,0.019600000232458115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,2047,0.016741332908471424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,2047,0.03376533339420954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,2047,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,2047,0.029338667790095013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,4095,0.016965333372354507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,4095,0.05426666637261709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,4095,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,4095,0.049914668003718056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,8191,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,8191,0.09639466802279155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,8191,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,8191,0.09126933415730794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,16383,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,16383,0.17909866571426392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,16383,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,16383,0.17102932929992676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,32767,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,32767,0.34432534376780194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,32767,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,32767,0.34725332260131836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,float16,65535,0.015775999675194424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,1,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,128,1,float16,fp8,65535,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,1,0.010586666564146677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,1,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,float16,65535,0.7755786577860514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,3,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,3,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,3,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,3,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,7,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,7,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,7,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,64,0,1,float16,fp8,65535,0.7449493408203125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,7,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,15,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,15,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,31,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,31,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,31,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,31,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,63,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,63,0.011744000017642975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,63,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,63,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,127,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,127,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,127,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,255,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,255,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,255,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,255,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,511,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,511,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,511,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,511,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,1023,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,1023,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,1023,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,1023,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,2047,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,2047,0.015376000354687372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,2047,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,2047,0.01624533285697301
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,4095,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,4095,0.017237332959969837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,4095,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,4095,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,8191,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,8191,0.01937066639463107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,8191,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,8191,0.01883200059334437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,16383,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,16383,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,16383,0.02144533395767212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,16383,0.021701333423455555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,32767,0.013365333278973898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,32767,0.02757866680622101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,32767,0.013904000322024027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,32767,0.025621332228183746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,float16,65535,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,float16,65535,0.04318400224049886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,128,1,float16,fp8,65535,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,1,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,1,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,1,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,3,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,3,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,3,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,64,0,1,float16,fp8,65535,0.03614933292071024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,7,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,3,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,7,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,7,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,15,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,15,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,15,0.011781333635250727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,31,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,31,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,31,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,63,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,63,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,63,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,63,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,127,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,127,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,255,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,255,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,255,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,255,0.012597333639860153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,511,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,511,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,511,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,1023,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,511,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,1023,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,1023,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,1023,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,2047,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,2047,0.013738666971524557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,2047,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,2047,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,4095,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,4095,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,4095,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,4095,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,8191,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,8191,0.01899733394384384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,8191,0.013946666071812311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,8191,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,16383,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,16383,0.02325333406527837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,16383,0.013653332988421122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,16383,0.021407999098300934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,32767,0.03796799977620443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,32767,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,32767,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,32767,0.03156800071398417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,float16,65535,0.01369599997997284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,float16,65535,0.05866133173306783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,128,1,float16,fp8,65535,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,1,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,64,0,1,float16,fp8,65535,0.05235733091831207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,1,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,3,0.011893333246310553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,3,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,3,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,7,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,7,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,7,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,15,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,15,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,15,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,31,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,31,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,63,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,63,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,63,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,127,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,127,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,127,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,255,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,255,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,255,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,255,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,511,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,511,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,511,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,511,0.01239466667175293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,1023,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,1023,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,1023,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,1023,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,2047,0.017173333714405697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,2047,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,2047,0.01340266689658165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,2047,0.0170666662355264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,4095,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,4095,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,4095,0.019424000134070713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,4095,0.01932266727089882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,8191,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,8191,0.02426133304834366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,8191,0.013408000270525614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,8191,0.022511998812357586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,16383,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,16383,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,16383,0.04036800066630045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,16383,0.03161599983771642
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,32767,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,32767,0.058517331878344216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,32767,0.013408000270525614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,32767,0.05226133267084757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,float16,65535,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,float16,65535,0.09963732957839966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,128,1,float16,fp8,65535,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,64,0,1,float16,fp8,65535,0.08708799878756206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,1,0.33267732461293537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,1,0.3339093526204427
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,1,0.2709119915962219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,1,0.2712426582972209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,3,0.33268266916275024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,3,0.3349013328552246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,3,0.2714773416519165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,3,0.2706986665725708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,7,0.34298133850097656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,7,0.3447680075963338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,7,0.28114134073257446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,7,0.2797226707140605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,15,0.3511999845504761
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,15,0.35124798615773517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,15,0.2925279935201009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,15,0.29209067424138385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,31,0.42742931842803955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,31,0.4273279905319214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,31,0.37231465180714923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,31,0.37191998958587646
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,63,0.4294133186340332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,63,0.42929601669311523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,63,0.3742613395055135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,63,0.3739413420359294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,127,0.4330879847208659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,127,0.43325865268707275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,127,0.3761813243230184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,127,0.3760053316752116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,float16,255,0.43303998311360675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,float16,255,0.5087093512217203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,128,1,float16,fp8,255,0.37626131375630695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,64,0,1,float16,fp8,255,0.4543733199437459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,1,0.029504001140594482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,1,0.02978666623433431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,1,0.03182400017976761
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,1,0.0315733328461647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,3,0.0295413335164388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,3,0.029824001093705494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,3,0.03160000095764796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,3,0.03161599983771642
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,7,0.029696000119050343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,7,0.030026666820049286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,7,0.0310506671667099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,7,0.031397332747777305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,15,0.030095999439557392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,15,0.029861333469549816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,15,0.03134933362404505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,15,0.03169066707293192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,31,0.029866665601730347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,31,0.029872000217437744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,31,0.03139200061559677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,31,0.031680000325044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,63,0.029701332251230877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,63,0.030031998952229817
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,63,0.03178666780392329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,63,0.031471999982992806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,127,0.029845332105954487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,127,0.029824001093705494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,127,0.031141333281993866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,127,0.03162666658560435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,float16,255,0.031983998914559685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,float16,255,0.02980799973011017
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,128,1,float16,fp8,255,0.031685332457224526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,1,0.041936000188191734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,64,0,1,float16,fp8,255,0.031343999008337654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,1,0.04262933135032654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,1,0.0421013335386912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,1,0.041450666884581246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,3,0.04178133110205332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,3,0.04212800165017446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,3,0.04170133173465729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,3,0.04177066683769226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,7,0.04161600023508072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,7,0.04215999941031138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,7,0.04185600082079569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,7,0.04174399872620901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,15,0.04189866781234741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,15,0.04172799984614054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,15,0.042405332128206887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,15,0.04190400242805481
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,31,0.04218666752179464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,31,0.0418453315893809
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,31,0.042266666889190674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,31,0.04188799858093262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,63,0.04173333446184794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,63,0.04215999941031138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,63,0.04200533529122671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,63,0.04185600082079569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,127,0.04186666508515676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,127,0.041663999358812966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,127,0.04188799858093262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,127,0.04189866781234741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,float16,255,0.044154668847719826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,128,1,float16,fp8,255,0.04188799858093262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,float16,255,0.04472533365090688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,1,0.07099199791749318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,64,0,1,float16,fp8,255,0.041877334316571556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,1,0.06821333368619283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,1,0.0702400008837382
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,1,0.06854400038719177
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,3,0.0703413337469101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,3,0.07025066514809926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,3,0.06833600004514058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,3,0.06883733471234639
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,7,0.07015466690063477
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,7,0.07063466807206471
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,7,0.06870933373769124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,7,0.06861866513888042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,15,0.07030933101971944
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,15,0.07025066514809926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,15,0.06858666737874348
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,15,0.06874666611353557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,31,0.07042133311430614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,31,0.07054399947325389
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,31,0.06870399912198384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,31,0.0684746652841568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,63,0.07041066884994507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,63,0.07017600039641063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,63,0.06851199766000111
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,63,0.06840000053246816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,127,0.07254933317502339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,127,0.0713973343372345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,127,0.06876799960931142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,127,0.06862933437029521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,float16,255,0.0767146646976471
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,float16,255,0.07315200070540111
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,128,1,float16,fp8,255,0.07208533088366191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,64,0,1,float16,fp8,255,0.07186666627724965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,1,0.6593973239262899
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,1,0.5341920057932535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,1,0.5336639881134033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,1,0.6584320068359375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,3,0.6598453521728516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,3,0.5341226657231649
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,3,0.5336533387502035
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,3,0.6593066851298014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,7,0.6809120178222656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,7,0.5531466801961263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,7,0.6808053652445475
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,7,0.5535840193430582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,15,0.6952319939931234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,15,0.6952319939931234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,15,0.5770826737085978
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,15,0.5779840151468912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,31,0.845088005065918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,31,0.8461439609527588
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,31,0.7373812993367513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,31,0.7368480364481608
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,63,0.8516746362050375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,63,0.8515786329905192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,63,0.7424213091532389
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,float16,127,0.8580853144327799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,63,0.7427732944488525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,float16,127,0.8635466893513998
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,1,0.05012799799442291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,128,1,float16,fp8,127,0.745136022567749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,64,0,1,float16,fp8,127,0.7450346946716309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,1,0.049738665421803795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,1,0.050016000866889954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,1,0.05072533090909322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,3,0.0505920002857844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,3,0.0498986691236496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,3,0.05052266518274943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,3,0.05020800232887268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,7,0.05009600023428599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,7,0.04978133241335551
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,7,0.05056533217430115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,7,0.05009600023428599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,15,0.049866666396458946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,15,0.050842667619387306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,15,0.05113600194454193
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,31,0.05020800232887268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,31,0.05217599868774414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,31,0.050160000721613564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,63,0.04980800052483877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,63,0.04966933528582255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,63,0.05169066786766052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,15,0.05030933519204458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,63,0.05031466484069824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,127,0.049642667174339294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,float16,31,0.050613333781560264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,float16,127,0.05153066913286845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,128,1,float16,fp8,127,0.05039466420809428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,64,0,1,float16,fp8,127,0.05189333359400431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,1,0.07648000121116638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,1,0.07504533231258392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,1,0.07309866448243459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,1,0.07471466561158498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,3,0.07664533456166585
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,3,0.07606933514277141
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,3,0.07292800148328145
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,3,0.07421866556008656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,7,0.07665599882602692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,7,0.07543466488520305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,7,0.0749120016892751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,15,0.075162669022878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,15,0.07730133334795634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,15,0.0734986662864685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,15,0.07457066575686137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,31,0.07572799921035767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,31,0.07544533411661784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,31,0.07457066575686137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,31,0.07371733089288075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,63,0.07525333265463512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,63,0.0748533308506012
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,63,0.07327466706434886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,7,0.07539199789365132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,63,0.07295999924341838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,float16,127,0.07704533139864604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,float16,127,0.0774239997069041
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,128,1,float16,fp8,127,0.075573335091273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,64,0,1,float16,fp8,127,0.0751200020313263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,1,0.13270399967829385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,1,0.13355732957522073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,1,0.12880000472068787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,3,0.1325973371664683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,1,0.12844799955685934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,3,0.13404267032941183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,3,0.12870400150616965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,3,0.12796266873677573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,7,0.13357866803805032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,7,0.13472533226013184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,7,0.12875733772913614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,7,0.1280693312486013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,15,0.13340800007184347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,15,0.1341386636098226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,15,0.12867732842763266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,15,0.1281653344631195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,31,0.13267733653386435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,31,0.1325546701749166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,31,0.12882133324941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,31,0.12801067034403482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,63,0.13276267051696777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,63,0.1329973340034485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,63,0.12879467010498047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,63,0.1281760036945343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,float16,127,0.13395733634630838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,1,0.01349866638580958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,1,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,fp8,127,0.13063466548919678
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,0,1,float16,fp8,127,0.1300320029258728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,1,0.01971199984351794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,3,0.01471466695268949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,1,0.015552000453074774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,3,0.014773332824309668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,3,0.015850666910409927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,3,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,7,0.01392000044385592
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,7,0.013631999492645264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,7,0.01563199982047081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,7,0.015642666568358738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,15,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,15,0.01551466683546702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,15,0.013552000125249227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,15,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,31,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,31,0.014885333677132925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,31,0.01695999999841054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,31,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,63,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,63,0.020842666427294414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,63,0.01551466683546702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,63,0.02046400060256322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,127,0.015344000111023584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,127,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,127,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,127,0.02139200021823247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,255,0.015477333217859268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,255,0.017450666675964992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,255,0.021349333226680756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,511,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,255,0.021397332350413006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,511,0.021290667355060577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,511,0.022101332743962605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,511,0.02350933353106181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,1023,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,1023,0.03141333411137263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,1023,0.03377600014209747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,1023,0.02170666555563609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,2047,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,2047,0.05416533350944519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,2047,0.02080533280968666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,4095,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,2047,0.05492266515890757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,4095,0.09523199995358785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,4095,0.021573332448800404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,4095,0.09316266576449077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,8191,0.01617066686352094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,8191,0.17987734079360962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,8191,0.021151999632517498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,8191,0.17089066902796426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,64,128,1,float16,float16,127,0.13379200299580893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,16383,0.015471999843915304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,16383,0.3546186685562134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,16383,0.02242133269707362
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,16383,0.3278026580810547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,float16,32767,0.01553600033124288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,1,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,128,1,float16,fp8,32767,0.019882666567961376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,1,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,1,0.011701333026091257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,1,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,3,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,3,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,3,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,3,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,float16,32767,0.9681759675343832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,64,0,1,float16,fp8,32767,0.6833173433939616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,7,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,7,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,7,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,15,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,31,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,31,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,63,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,63,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,63,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,127,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,63,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,127,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,127,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,127,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,255,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,255,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,255,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,255,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,511,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,511,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,511,0.012752000242471695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,1023,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,1023,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,1023,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,1023,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,2047,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,2047,0.015360000232855478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,2047,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,2047,0.015333333363135656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,4095,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,4095,0.017093333105246227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,4095,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,4095,0.016832000265518825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,8191,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,8191,0.021274665991465252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,8191,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,8191,0.01926933353145917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,16383,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,16383,0.027285332481066387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,16383,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,16383,0.025466665625572205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,float16,32767,0.013445333888133367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,float16,32767,0.043680002291997276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,1,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,128,1,float16,fp8,32767,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,1,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,1,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,1,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,64,0,1,float16,fp8,32767,0.03573333223660787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,3,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,3,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,3,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,7,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,7,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,7,0.01071999967098236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,15,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,15,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,15,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,15,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,31,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,31,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,31,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,63,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,63,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,63,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,127,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,255,0.010575999816258749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,255,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,255,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,255,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,511,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,511,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,511,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,511,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,1023,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,1023,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,1023,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,1023,0.012709333250919977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,2047,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,2047,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,2047,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,2047,0.016821333517630894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,4095,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,4095,0.019445333629846573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,4095,0.013541333377361298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,4095,0.019152000546455383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,8191,0.013717333475748697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,8191,0.0136266661187013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,8191,0.02550933261712392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,8191,0.02342933416366577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,16383,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,16383,0.039733332892258964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,16383,0.01441066712141037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,16383,0.03159466634194056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,float16,32767,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,float16,32767,0.059808000922203064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,1,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,128,1,float16,fp8,32767,0.013855999956528345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,64,0,1,float16,fp8,32767,0.05202666421731313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,1,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,1,0.011802667131026586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,1,0.01192533348997434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,3,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,3,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,3,0.0120319997270902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,7,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,7,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,3,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,15,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,15,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,15,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,15,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,31,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,31,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,31,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,31,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,63,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,63,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,63,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,63,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,127,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,127,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,255,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,255,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,255,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,255,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,511,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,511,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,511,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,511,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,1023,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,1023,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,1023,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,1023,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,2047,0.014773332824309668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,2047,0.018112000077962875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,2047,0.01820266619324684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,2047,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,4095,0.013445333888133367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,4095,0.023578666150569916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,4095,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,4095,0.021850667893886566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,8191,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,8191,0.03782399992148081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,8191,0.013568000247081121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,8191,0.02972800036271413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,16383,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,16383,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,16383,0.05861333509286245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,16383,0.052469333012898765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,float16,32767,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,float16,32767,0.0997173289457957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,1,0.017407999684413273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,128,1,float16,fp8,32767,0.014794666320085526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,1,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,3,0.01846933364868164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,64,0,1,float16,fp8,32767,0.08503466844558716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,3,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,7,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,7,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,15,0.019632000476121902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,15,0.017632000148296356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,31,0.019391999890406925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,31,0.017173333714405697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,63,0.01979200045267741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,63,0.01874133323629697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,127,0.02309333284695943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,127,0.021253332495689392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,255,0.02951466788848241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,255,0.031925333042939506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,511,0.05036800106366476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,511,0.04786133269468943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,1023,0.08504000306129456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,1023,0.0828959991534551
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,2047,0.1544426679611206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,2047,0.15288533767064413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,4095,0.29638399680455524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,4095,0.2916799982388814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,8191,0.5891786813735962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,8191,0.5732053518295288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,1,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,3,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,1,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,7,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,3,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,7,0.012634667257467905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,15,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,31,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,15,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,float16,16383,1.4124693870544434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,63,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,1,128,0,1,float16,fp8,16383,1.4574774106343586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,127,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,63,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,255,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,511,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,511,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,1023,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,1023,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,2047,0.01932799940307935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,2047,0.02070933332045873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,4095,0.023210667073726654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,4095,0.021269333859284718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,8191,0.027466667195161183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,8191,0.040805332362651825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,1,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,3,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,float16,16383,0.060517330964406334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,2,128,0,1,float16,fp8,16383,0.04275733232498169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,7,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,7,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,15,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,31,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,15,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,31,0.012378666549921036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,63,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,63,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,127,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,127,0.012442667037248611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,255,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,511,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,511,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,1023,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,1023,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,2047,0.021269333859284718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,2047,0.019152000546455383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,4095,0.039749334255854286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,4095,0.023402666052182514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,8191,0.05871466795603434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,8191,0.03872533390919367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,1,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,float16,16383,0.10003200173377991
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,4,128,0,1,float16,fp8,16383,0.05753066639105479
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,1,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,3,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,3,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,7,0.013573333621025085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,7,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,15,0.01268799975514412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,15,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,31,0.013744000345468521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,31,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,63,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,63,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,127,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,127,0.014650666465361914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,255,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,255,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,511,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,511,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,1023,0.019461333751678467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,1023,0.016997333616018295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,2047,0.03921066721280416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,2047,0.02571733295917511
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,4095,0.059664001067479454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,4095,0.039962666730086006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,8191,0.10178132851918538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,8191,0.06084266801675161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,1,0.008896000062425932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,1,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,3,0.009173333023985228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,float16,16383,0.18331732352574667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,3,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,32,8,128,0,1,float16,fp8,16383,0.10136000315348308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,7,0.009365333244204521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,7,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,15,0.009296000003814697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,31,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,63,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,31,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,63,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,127,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,127,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,255,0.014736000448465347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,255,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,511,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,511,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,1023,0.01894933357834816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,1023,0.021168000996112823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,2047,0.027674667537212372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,2047,0.029077333708604176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,4095,0.037589333951473236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,4095,0.03802666564782461
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,8191,0.05586666862169901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,8191,0.0543093333641688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,16383,0.08454400300979614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,16383,0.0926080048084259
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,32767,0.1666719913482666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,32767,0.14451199769973755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,1,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,1,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,3,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,7,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,7,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,15,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,float16,65535,0.31267199913660687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,1,128,0,1,float16,fp8,65535,0.2685920000076294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,31,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,31,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,63,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,63,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,127,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,255,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,255,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,511,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,511,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,1023,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,1023,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,2047,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,2047,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,4095,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,4095,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,8191,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,8191,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,16383,0.02319466571013133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,16383,0.021386665602525074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,32767,0.02566933383544286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,32767,0.025466665625572205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,1,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,float16,65535,0.02718399961789449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,1,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,3,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,3,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,7,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,7,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,15,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,2,128,0,1,float16,fp8,65535,0.02533866713444392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,31,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,63,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,63,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,127,0.010608000059922537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,127,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,255,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,511,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,511,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,1023,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,1023,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,2047,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,2047,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,4095,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,4095,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,8191,0.016986666868130367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,8191,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,16383,0.019199999670187633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,16383,0.01882133384545644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,32767,0.0220266655087471
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,32767,0.01937599976857503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,1,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,float16,65535,0.02332799881696701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,1,0.011658667276302973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,3,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,4,128,0,1,float16,fp8,65535,0.02142400046189626
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,15,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,15,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,31,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,31,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,63,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,63,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,127,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,255,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,127,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,255,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,511,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,511,0.011866666376590729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,1023,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,1023,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,2047,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,2047,0.012543999900420507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,4095,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,4095,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,8191,0.01692266638080279
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,8191,0.01720533271630605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,16383,0.019744000087181728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,16383,0.017759999881188076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,32767,0.019248000035683315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,32767,0.023989332218964893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,1,0.009183999771873156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,3,0.00921066664159298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,1,0.012245333443085352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,3,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,7,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,7,0.010224000240365664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,float16,65535,0.03836799909671148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,15,0.009125333279371262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,32,8,128,0,1,float16,fp8,65535,0.02517866591612498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,15,0.011594666788975397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,31,0.01055466632048289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,31,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,63,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,63,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,127,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,255,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,255,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,511,0.021615999440352123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,511,0.017530667285124462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,1023,0.025434667865435284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,1023,0.029520000020662945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,2047,0.0335413341720899
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,2047,0.035749333600203194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,4095,0.04943466683228811
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,4095,0.0483893354733785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,8191,0.08062399923801422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,8191,0.07492266595363617
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,16383,0.14244799812634787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,16383,0.12566933035850525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,32767,0.26393600304921466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,32767,0.22866666316986084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,1,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,1,0.011781333635250727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,float16,65535,0.5094559987386068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,3,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,7,0.01062400018175443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,3,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,15,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,7,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,1,128,0,1,float16,fp8,65535,0.43145068486531574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,15,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,31,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,31,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,63,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,63,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,127,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,127,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,255,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,511,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,255,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,511,0.01360000049074491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,1023,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,1023,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,2047,0.013503999759753546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,2047,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,4095,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,4095,0.014309333016475042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,8191,0.019280000279347103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,8191,0.020015999674797058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,16383,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,16383,0.021525333325068157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,32767,0.02348800003528595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,32767,0.022976001103719074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,1,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,float16,65535,0.027306665976842243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,2,128,0,1,float16,fp8,65535,0.025519999365011852
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,15,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,15,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,31,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,31,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,63,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,63,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,127,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,255,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,255,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,511,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,511,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,1023,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,1023,0.012096000214417776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,2047,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,2047,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,4095,0.01602666700879733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,4095,0.017082666357358296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,8191,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,8191,0.017562666287024815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,16383,0.01877333347996076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,16383,0.021712000171343487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,32767,0.023226665953795116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,32767,0.0207893339296182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,1,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,1,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,3,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,3,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,7,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,7,0.01192533348997434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,float16,65535,0.037690666814645134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,15,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,15,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,31,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,4,128,0,1,float16,fp8,65535,0.025450666745503742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,31,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,63,0.010602666685978571
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,127,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,127,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,255,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,255,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,511,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,511,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,1023,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,1023,0.012736000120639801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,2047,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,2047,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,4095,0.015583999454975128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,4095,0.017162666966517765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,8191,0.01770666614174843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,8191,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,16383,0.022287999590237934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,16383,0.019285333653291065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,32767,0.03750933210055033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,32767,0.023685333629449207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,1,0.02773333340883255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,1,0.02489600082238515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,float16,65535,0.05657599866390228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,32,8,128,0,1,float16,fp8,65535,0.03746666759252548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,3,0.028016000986099243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,3,0.02495466669400533
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,7,0.02537599951028824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,7,0.027514666318893433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,15,0.03223466624816259
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,15,0.029989334444204967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,31,0.033386667569478355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,31,0.029487999776999157
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,63,0.03326933334271113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,63,0.030767999589443207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,127,0.03788266579310099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,127,0.03602133442958196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,255,0.056458666920661926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,255,0.052890668312708534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,511,0.09080533186594646
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,511,0.08748799562454224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,1023,0.15442132949829102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,1023,0.1593706707159678
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,2047,0.29848533868789673
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,2047,0.28892266750335693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,4095,0.5773280064264933
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,4095,0.556826670964559
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,1,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,1,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,float16,8191,1.160703976949056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,3,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,7,0.011493333925803503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,7,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,1,128,0,1,float16,fp8,8191,1.101914644241333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,15,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,15,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,31,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,31,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,63,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,63,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,127,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,127,0.013349333157142004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,255,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,511,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,255,0.013642666240533194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,511,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,1023,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,1023,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,2047,0.020997333029905956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,2047,0.022197333474953968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,4095,0.026602665583292644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,4095,0.03995733211437861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,1,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,1,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,float16,8191,0.060496002435684204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,2,128,0,1,float16,fp8,8191,0.042117332418759666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,3,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,3,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,7,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,7,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,15,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,15,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,31,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,31,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,63,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,63,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,127,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,127,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,255,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,255,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,511,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,511,0.015370666980743408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,1023,0.018373332917690277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,1023,0.016943999876578648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,2047,0.04008000095685323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,2047,0.025226667523384094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,4095,0.04012266546487808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,4095,0.060496002435684204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,1,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,1,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,float16,8191,0.1006773312886556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,4,128,0,1,float16,fp8,8191,0.06043733159701029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,3,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,3,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,7,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,7,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,15,0.01471466695268949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,15,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,31,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,31,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,63,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,63,0.014741333822409311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,127,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,127,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,255,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,255,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,511,0.019088000059127808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,511,0.017488000293572743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,1023,0.03570133447647095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,1023,0.02128533273935318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,2047,0.06084266801675161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,2047,0.041093334555625916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,4095,0.10131200154622395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,4095,0.06241600215435028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,1,0.009408000235756239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,float16,8191,0.18616533279418945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,1,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,3,0.009445333232482275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,3,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,32,8,128,0,1,float16,fp8,8191,0.10193066795667012
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,7,0.009338666374484697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,15,0.009317333499590555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,15,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,31,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,63,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,63,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,127,0.013690666606028875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,255,0.021733333667119343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,255,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,511,0.02347733328739802
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,511,0.02752533306678136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,1023,0.02991466720898946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,1023,0.03169066707293192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,2047,0.0422026664018631
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,2047,0.04234133164087931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,4095,0.0651093324025472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,4095,0.062218666076660156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,8191,0.11158399780591328
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,8191,0.09982933600743611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,16383,0.2038559913635254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,16383,0.17755200465520224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,32767,0.3884906768798828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,32767,0.3303946654001872
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,1,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,3,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,7,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,7,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,15,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,float16,65535,0.7630613644917806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,15,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,31,0.011882666498422623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,31,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,1,128,0,1,float16,fp8,65535,0.6362346808115641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,63,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,63,0.011637333780527115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,127,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,255,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,255,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,511,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,511,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,1023,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,1023,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,2047,0.015509333461523056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,2047,0.015376000354687372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,4095,0.01939733326435089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,4095,0.019050666441520054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,8191,0.020975999534130096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,8191,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,16383,0.021317332983016968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,16383,0.024245334168275196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,32767,0.025114665428797405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,32767,0.023317334552605946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,1,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,3,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,1,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,3,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,7,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,7,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,15,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,float16,65535,0.04258666435877482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,31,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,31,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,2,128,0,1,float16,fp8,65535,0.02940800040960312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,63,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,63,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,127,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,127,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,255,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,511,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,511,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,1023,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,1023,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,2047,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,2047,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,4095,0.017551999539136887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,4095,0.017317333569129307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,8191,0.01940800001223882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,8191,0.0189280000825723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,16383,0.02362666775782903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,16383,0.02142400046189626
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,32767,0.041402667760849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,32767,0.025733334322770435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,1,0.011920000116030375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,1,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,float16,65535,0.05964266757170359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,3,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,3,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,7,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,7,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,15,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,4,128,0,1,float16,fp8,65535,0.039461334546407066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,31,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,31,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,63,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,127,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,127,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,255,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,511,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,511,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,1023,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,1023,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,2047,0.01533866673707962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,2047,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,4095,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,4095,0.017071999609470367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,8191,0.021381333470344543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,8191,0.018901333212852478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,16383,0.03935466706752777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,16383,0.02386133372783661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,32767,0.058362667759259544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,32767,0.03828799972931544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,1,0.04588800172011057
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,1,0.04018666595220566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,float16,65535,0.10029866298039754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,3,0.048207998275756836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,32,8,128,0,1,float16,fp8,65535,0.05671999851862589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,3,0.04154133299986521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,7,0.04808533191680908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,7,0.043050666650136314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,15,0.05815466741720835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,15,0.05182399849096934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,31,0.0582826683918635
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,31,0.05236800014972687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,63,0.05821866790453593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,63,0.053082664807637535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,127,0.06846933563550313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,255,0.10132267077763875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,127,0.0646613339583079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,255,0.09539199868837993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,511,0.1665493349234263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,511,0.15918933351834616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,1023,0.2876960039138794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,1023,0.2992639938990275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,2047,0.5682559808095297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,2047,0.542143980662028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,1,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,1,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,3,0.01461333284775416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,3,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,7,0.014538666854302088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,7,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,float16,4095,1.1166186332702637
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,15,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,1,128,0,1,float16,fp8,4095,1.0715573628743489
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,15,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,31,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,31,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,63,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,63,0.013557333499193192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,127,0.013552000125249227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,127,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,255,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,255,0.013477332890033722
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,511,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,511,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,1023,0.019082666685183842
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,1023,0.01889066646496455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,2047,0.02769600103298823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,2047,0.040976000328858696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,1,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,1,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,float16,4095,0.061333333452542625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,2,128,0,1,float16,fp8,4095,0.044064000248909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,3,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,3,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,7,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,7,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,15,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,15,0.015626666446526844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,31,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,31,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,63,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,63,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,127,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,127,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,255,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,255,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,511,0.019226666539907455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,511,0.017743999759356182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,1023,0.035546667873859406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,1023,0.023306667804718018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,2047,0.0621973325808843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,2047,0.04178133110205332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,1,0.019178666174411774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,float16,4095,0.10266666611035664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,4,128,0,1,float16,fp8,4095,0.06223999957243601
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,1,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,3,0.019359999646743137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,3,0.0185759998857975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,7,0.019018666197856266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,7,0.01874133323629697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,15,0.019466667125622433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,15,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,31,0.019007999449968338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,31,0.018085333208243053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,63,0.01904533306757609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,63,0.018383999665578205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,127,0.01889066646496455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,127,0.018976000448067982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,255,0.01940800001223882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,255,0.018570666511853535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,511,0.03373866776625315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,511,0.022965334355831146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,1023,0.05221866567929586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,1023,0.035589332381884255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,2047,0.09531733393669128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,2047,0.05823466678460439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,float16,4095,0.17157334089279175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,1,0.08694932858149211
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,32,8,128,0,1,float16,fp8,4095,0.09468799829483032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,1,0.07454933226108551
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,3,0.09065600236256917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,3,0.07660266757011414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,7,0.09092266360918681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,7,0.07913066446781158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,15,0.10964266459147136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,15,0.0981280008951823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,31,0.1099626620610555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,31,0.09921066959698994
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,63,0.10932266712188721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,63,0.0995146632194519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,127,0.13013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,255,0.1952000061670939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,255,0.1832853356997172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,127,0.12057600418726604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,511,0.32468267281850177
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,511,0.31060266494750977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,1,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,1,0.016970666746298473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,3,0.017530667285124462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,float16,1023,0.5983519951502482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,1,128,0,1,float16,fp8,1023,0.5623573462168375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,3,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,7,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,7,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,15,0.017103999853134155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,31,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,15,0.017674667139848072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,31,0.017317333569129307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,63,0.017338667064905167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,63,0.01693333312869072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,127,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,127,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,255,0.016762666404247284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,255,0.016949333250522614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,511,0.021456000705560047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,511,0.020341333001852036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,float16,1023,0.037690666814645134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,2,128,0,1,float16,fp8,1023,0.02534399926662445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,1,0.020981334149837494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,1,0.01912533367673556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,3,0.019189332922299702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,3,0.019167999426523846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,7,0.020021333048741024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,7,0.0194560003777345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,15,0.021104000508785248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,15,0.018725333114465077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,31,0.02012266715367635
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,31,0.019589333484570186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,63,0.02147199958562851
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,63,0.019280000279347103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,127,0.021221332252025604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,127,0.01942933350801468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,255,0.01926933353145917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,255,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,511,0.03544000039498011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,511,0.025216000775496166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,1,0.029530666768550873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,float16,1023,0.05442666510740916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,4,128,0,1,float16,fp8,1023,0.035616000493367515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,1,0.02718399961789449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,3,0.02942933390537898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,3,0.02514133354028066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,7,0.029557332396507263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,7,0.02718399961789449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,15,0.029370665550231934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,15,0.02734400083621343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,31,0.029472000896930695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,31,0.025600001215934753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,63,0.029717333614826202
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,63,0.027317332724730175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,127,0.02847466617822647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,127,0.02565866708755493
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,255,0.03526933242877325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,255,0.025983999172846477
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,511,0.05462400118509928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,511,0.03854399919509888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,float16,1023,0.09325333436330159
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,32,8,128,0,1,float16,fp8,1023,0.055770665407180786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,1,0.16703999042510986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,1,0.14082666238149008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,3,0.17326400677363077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,3,0.14619200428326926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,7,0.1774453322092692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,7,0.15281066298484802
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,15,0.21388266483942667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,15,0.19005866845448813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,31,0.19112000862757364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,31,0.214026669661204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,63,0.191210667292277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,63,0.2145973245302836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,127,0.25355199972788495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,127,0.23213332891464233
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,255,0.3821066617965698
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,255,0.3596373399098714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,1,0.023344000180562336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,float16,511,0.6569493214289347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,1,0.02384000023206075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,1,128,0,1,float16,fp8,511,0.6121973196665446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,3,0.023562667270501454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,3,0.023376000424226124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,7,0.023728000621000927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,7,0.023503998915354412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,15,0.023669332265853882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,15,0.023317334552605946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,31,0.0236160010099411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,63,0.023381332556406658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,31,0.023503998915354412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,63,0.023381332556406658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,127,0.02370133250951767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,127,0.023498666783173878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,255,0.023472001155217487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,255,0.023344000180562336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,float16,511,0.03803733239571253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,2,128,0,1,float16,fp8,511,0.029301332930723827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,1,0.03142933299144109
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,1,0.029194665451844532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,3,0.031658666829268135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,7,0.03158933420976003
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,3,0.029680001238981884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,7,0.029279999434947968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,15,0.03161599983771642
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,15,0.029765332738558452
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,31,0.031712000568707786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,31,0.029264000554879505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,63,0.03166399896144867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,63,0.029509333272775013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,127,0.031557333966096245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,127,0.029552000264326733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,255,0.03736533224582672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,255,0.02938666691382726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,float16,511,0.056554665168126426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,4,128,0,1,float16,fp8,511,0.03977599988381068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,1,0.0480373352766037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,1,0.041989331444104515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,3,0.04823466638724009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,3,0.04199466605981191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,7,0.047541335225105286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,7,0.041989331444104515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,15,0.04785599807898203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,15,0.04189866781234741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,31,0.0480320006608963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,31,0.04182399809360504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,63,0.04755199948946635
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,63,0.042021334171295166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,127,0.04188266893227895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,127,0.050288001696268715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,255,0.05574400226275126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,1,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,255,0.04623466730117798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,3,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,fp8,511,0.06396799782911937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,32,8,128,0,1,float16,float16,511,0.09474133451779683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,7,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,15,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,15,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,31,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,31,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,63,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,63,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,127,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,127,0.01578666642308235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,255,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,255,0.01469333345691363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,511,0.023333333432674408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,511,0.01941866676012675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,1023,0.03379733363787333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,1023,0.029743999242782593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,2047,0.05444799860318502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,2047,0.05044800043106079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,4095,0.09540266791979472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,4095,0.09035733342170715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,8191,0.1798080007235209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,8191,0.16955200831095377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,16383,0.34337600072224933
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,16383,0.3291626572608948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,32767,0.6827413241068522
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,32767,0.7335360050201416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,1,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,1,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,3,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,7,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,7,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,15,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,31,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,31,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,float16,65535,1.73636261622111
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,63,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,63,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,127,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,127,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,255,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,255,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,1,128,0,1,float16,fp8,65535,1.5762292544047039
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,511,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,511,0.01369599997997284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,1023,0.012426666915416718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,1023,0.013365333278973898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,2047,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,4095,0.019109333554903667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,2047,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,4095,0.018474667022625606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,8191,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,8191,0.02197333425283432
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,16383,0.02513599892457326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,16383,0.023130667706330616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,32767,0.04207466542720795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,1,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,32767,0.027776000400384266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,1,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,3,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,3,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,7,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,7,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,float16,65535,0.06163200239340464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,15,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,15,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,2,128,0,1,float16,fp8,65535,0.044256001710891724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,31,0.011727999895811081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,31,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,63,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,63,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,127,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,255,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,255,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,511,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,511,0.012602667013804117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,1023,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,1023,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,2047,0.016986666868130367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,2047,0.015893333901961643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,4095,0.017386666188637417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,4095,0.017136000096797943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,8191,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,8191,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,16383,0.03972266614437103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,16383,0.02378133436044057
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,32767,0.06015466650327047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,32767,0.03808533400297165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,1,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,1,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,3,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,7,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,3,0.012069333344697952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,15,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,15,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,float16,65535,0.09866666793823242
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,4,128,0,1,float16,fp8,65535,0.058186665177345276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,31,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,63,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,127,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,127,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,255,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,255,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,511,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,511,0.011765333513418833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,1023,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,1023,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,2047,0.01758933315674464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,2047,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,4095,0.021344001094500225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,4095,0.019434666881958645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,8191,0.037962667644023895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,8191,0.02438933402299881
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,16383,0.06000000238418579
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,16383,0.03794133414824804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,32767,0.05844266712665558
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,32767,0.10142399867375691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,float16,65535,0.1821813384691874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,1,0.32708799839019775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,32,8,128,0,1,float16,fp8,65535,0.0976106623808543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,1,0.2734079957008362
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,3,0.2842400074005127
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,3,0.34138667583465576
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,7,0.34748268127441406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,7,0.2977386713027954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,15,0.4212799866994222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,15,0.3738880157470703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,31,0.42053866386413574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,31,0.3744639952977498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,63,0.3757760127385457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,127,0.5010986725489298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,63,0.42256001631418866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,127,0.45601598421732586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,1,0.036490666369597115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,1,0.037674665451049805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,float16,255,0.7839946746826172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,1,128,0,1,float16,fp8,255,0.7113706270853678
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,3,0.037402667105197906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,3,0.03669866671164831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,7,0.03789333254098892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,7,0.0363520011305809
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,15,0.03722666700681051
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,15,0.036490666369597115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,31,0.03723733375469843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,31,0.03566399961709976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,63,0.037258667250474296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,63,0.03620799879233042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,127,0.037674665451049805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,127,0.0355679988861084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,float16,255,0.042170668641726174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,1,0.05197333296140035
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,2,128,0,1,float16,fp8,255,0.03909866760174433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,1,0.047775998711586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,3,0.05264533559481303
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,3,0.04818133513132731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,7,0.052501335740089417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,7,0.04775466521581014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,15,0.052986666560173035
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,15,0.04781866570313772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,31,0.05266666909058889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,31,0.048138668139775596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,63,0.05266133447488149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,63,0.04821866750717163
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,127,0.05458133419354757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,127,0.04782933493455251
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,float16,255,0.06197333335876465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,4,128,0,1,float16,fp8,255,0.051738664507865906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,1,0.08719467123349507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,1,0.07659733295440674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,3,0.08690667152404785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,3,0.07674666742483775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,7,0.08733333150545756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,7,0.0765119989713033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,15,0.07667733232180278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,15,0.08685333530108134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,31,0.08703466256459554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,31,0.07647466659545898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,63,0.08725333213806152
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,63,0.07633066674073537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,127,0.08726933598518372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,127,0.0784693310658137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,fp8,255,0.08092266817887624
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,32,8,128,0,1,float16,float16,255,0.09961600104967754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,1,0.6445493300755819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,1,0.5375200112660726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,3,0.5583253304163615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,3,0.6732587019602457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,7,0.5872746706008911
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,7,0.686837355295817
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,15,0.8322133223215739
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,15,0.7383946577707926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,31,0.8329813480377197
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,31,0.7401920159657797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,63,0.7422719796498617
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,63,0.8399360179901123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,fp8,127,0.91047469774882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,1,128,0,1,float16,float16,127,1.034325361251831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,1,0.06353066861629486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,1,0.06025599936644236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,3,0.06292800108591716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,3,0.059978668888409935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,7,0.06399466594060262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,7,0.06016000111897787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,15,0.06334400177001953
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,15,0.060362666845321655
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,31,0.06418666740258534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,63,0.06356800099213918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,31,0.06019733349482218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,63,0.060922667384147644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,float16,127,0.06393066545327504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,1,0.09472533067067464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,2,128,0,1,float16,fp8,127,0.06278933087984721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,1,0.08683733145395915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,3,0.09474133451779683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,3,0.08577066659927368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,7,0.09418666362762451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,15,0.09436800082524617
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,15,0.08627733588218689
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,7,0.08661333719889323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,31,0.09425600369771321
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,31,0.08720533053080241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,63,0.09390399853388469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,63,0.08574400345484416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,float16,127,0.09613866607348125
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,1,0.16079466541608176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,4,128,0,1,float16,fp8,127,0.08705600102742513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,1,0.14265599846839905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,3,0.16125333309173584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,3,0.14299733440081278
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,7,0.1609546641508738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,7,0.1423466702302297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,15,0.16142400105794272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,15,0.14243200421333313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,31,0.1612106661001841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,31,0.1423786679903666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,63,0.16081066926320395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,63,0.14265066385269165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,1,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,float16,127,0.1622666617234548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,3,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,1,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,3,0.015477333217859268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,7,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,7,0.01523200049996376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,15,0.015381333728631338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,15,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,31,0.014778666198253632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,31,0.019285333653291065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,32,8,128,0,1,float16,fp8,127,0.1423466702302297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,63,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,63,0.019658666104078293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,127,0.01664000004529953
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,127,0.020261333634455998
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,255,0.021221332252025604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,255,0.02365333338578542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,511,0.031845333675543465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,511,0.03350933392842611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,1023,0.05223466455936432
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,1023,0.054101333022117615
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,2047,0.09521599610646565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,2047,0.09159466624259949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,4095,0.17922665675481161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,4095,0.1694399913152059
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,8191,0.34557334582010907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,8191,0.3240373333295186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,16383,0.6319359938303629
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,16383,0.6918719609578451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,1,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,1,0.012213333199421564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,3,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,3,0.012175999581813812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,7,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,7,0.011861333002646765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,15,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,15,0.012357333054145178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,31,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,31,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,63,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,63,0.012639999389648438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,127,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,fp8,32767,1.4606293042500813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,255,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,511,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,255,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,511,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,1023,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,1023,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,2047,0.017077332983414333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,2047,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,1,128,0,1,float16,float16,32767,2.4344746271769204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,4095,0.019248000035683315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,4095,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,8191,0.02319466571013133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,8191,0.021242665747801464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,16383,0.0402399996916453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,16383,0.027258666853109997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,1,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,3,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,float16,32767,0.06046399970849355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,7,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,7,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,2,128,0,1,float16,fp8,32767,0.04314666489760081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,15,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,31,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,63,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,63,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,127,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,127,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,255,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,255,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,511,0.012330666184425354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,511,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,1023,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,1023,0.013781332721312841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,2047,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,2047,0.01932799940307935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,4095,0.021888000269730885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,4095,0.019359999646743137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,8191,0.040037333965301514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,8191,0.026005332668622334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,16383,0.059936001896858215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,16383,0.039279999832312264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,1,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,1,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,float16,32767,0.10035199920336406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,4,128,0,1,float16,fp8,32767,0.05853866537412008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,3,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,7,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,3,0.011866666376590729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,7,0.011829332758982977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,15,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,15,0.011749333391586939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,31,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,63,0.011621333658695221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,127,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,63,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,127,0.012378666549921036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,255,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,255,0.012047999848922094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,511,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,511,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,1023,0.01579733317097028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,1023,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,2047,0.019904000063737232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,2047,0.020015999674797058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,4095,0.037978666524092354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,4095,0.02367999901374181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,8191,0.05949333310127258
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,8191,0.03769599894682566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,16383,0.09985066453615825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,16383,0.05867200096448263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,1,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,float16,32767,0.1814026633898417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,1,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,3,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,32,8,128,0,1,float16,fp8,32767,0.0971999963124593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,3,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,3,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,7,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,7,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,7,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,15,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,7,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,15,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,15,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,31,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,15,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,31,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,31,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,63,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,63,0.012351999680201212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,63,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,63,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,127,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,127,0.011621333658695221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,127,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,127,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,255,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,255,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,255,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,255,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,511,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,511,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,511,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,1023,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,511,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,1023,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,1023,0.012719999998807907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,1023,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,2047,0.01357866699496905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,2047,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,2047,0.014698666830857595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,2047,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,4095,0.014096000542243322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,4095,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,4095,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,4095,0.01815466706951459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,8191,0.02102400114138921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,8191,0.013487999637921652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,8191,0.020629333953062694
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,8191,0.014181333283583323
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,16383,0.013418667018413544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,16383,0.02719466636578242
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,16383,0.01350933313369751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,16383,0.025402667621771496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,float16,32767,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,float16,32767,0.04368533194065094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,128,1,float16,fp8,32767,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,1,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,1,0.011855999628702799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,1,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,64,0,1,float16,fp8,32767,0.03570666660865148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,3,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,3,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,3,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,3,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,7,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,7,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,7,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,15,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,15,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,15,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,31,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,31,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,63,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,63,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,63,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,127,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,127,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,127,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,127,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,255,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,255,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,255,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,255,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,511,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,511,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,511,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,511,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,1023,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,1023,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,1023,0.011701333026091257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,1023,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,2047,0.014655999839305878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,2047,0.014789332946141561
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,2047,0.017903999735911686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,2047,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,4095,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,4095,0.019194666296243668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,4095,0.019365333020687103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,4095,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,8191,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,8191,0.025583999852339428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,8191,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,8191,0.02384000023206075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,16383,0.039887999494870506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,16383,0.013557333499193192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,16383,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,16383,0.03224000086386999
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,float16,32767,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,float16,32767,0.0610453337430954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,1,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,128,1,float16,fp8,32767,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,64,0,1,float16,fp8,32767,0.05258666475613912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,1,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,1,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,3,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,1,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,3,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,3,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,3,0.01210133358836174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,7,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,7,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,7,0.01209066684047381
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,15,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,15,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,15,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,31,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,15,0.012026666353146235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,31,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,63,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,63,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,63,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,127,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,127,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,127,0.011493333925803503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,127,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,255,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,255,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,255,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,511,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,511,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,511,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,511,0.012282667060693106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,1023,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,1023,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,1023,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,1023,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,2047,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,2047,0.018853332847356796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,2047,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,2047,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,4095,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,4095,0.023434666295846302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,4095,0.023082666099071503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,4095,0.015647999942302704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,8191,0.01482133318980535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,8191,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,8191,0.038986665507157646
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,8191,0.03166933357715607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,16383,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,16383,0.0582826683918635
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,16383,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,16383,0.05230933427810669
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,float16,32767,0.013989333063364029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,float16,32767,0.1013866662979126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,1,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,1,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,128,1,float16,fp8,32767,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,1,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,1,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,3,0.013530666629473368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,3,0.011802667131026586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,64,0,1,float16,fp8,32767,0.08660266796747844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,3,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,3,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,7,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,7,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,7,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,7,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,15,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,15,0.011823999385039011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,15,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,15,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,31,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,31,0.012117333710193634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,31,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,31,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,63,0.012448000411192576
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,63,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,63,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,63,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,127,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,127,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,127,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,255,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,255,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,127,0.013605333864688873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,255,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,255,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,511,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,511,0.014426667243242264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,511,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,511,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,1023,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,1023,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,1023,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,1023,0.015418666104475657
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,2047,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,2047,0.023749334116776783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,2047,0.023061332603295643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,2047,0.015647999942302704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,4095,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,4095,0.038975998759269714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,4095,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,4095,0.031930667658646904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,8191,0.015520000209410986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,8191,0.060309335589408875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,8191,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,8191,0.0526506652434667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,16383,0.015450666348139444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,16383,0.1030346651871999
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,16383,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,16383,0.08690133690834045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,float16,32767,0.01543466622630755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,float16,32767,0.18628267447153726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,128,1,float16,fp8,32767,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,1,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,1,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,64,0,1,float16,fp8,32767,0.15486933787663779
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,1,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,1,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,3,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,3,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,3,0.011760000139474869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,3,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,7,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,7,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,7,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,7,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,15,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,15,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,15,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,15,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,31,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,31,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,31,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,31,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,63,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,63,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,63,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,63,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,127,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,127,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,127,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,127,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,255,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,255,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,255,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,255,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,511,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,511,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,511,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,511,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,1023,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,1023,0.011952000359694162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,1023,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,1023,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,2047,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,2047,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,2047,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,2047,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,4095,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,4095,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,4095,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,4095,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,8191,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,8191,0.016341333587964375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,8191,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,8191,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,16383,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,16383,0.019285333653291065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,16383,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,16383,0.01907733331123988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,32767,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,32767,0.029232000311215717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,32767,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,32767,0.025466665625572205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,65535,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,65535,0.03332799921433131
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,65535,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,65535,0.0317493329445521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,float16,131071,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,float16,131071,0.035418666899204254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,1,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,1,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,3,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,3,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,0,1,float16,fp8,131071,0.03450666616360346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,3,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,7,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,64,128,1,float16,fp8,131071,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,7,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,7,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,7,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,15,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,15,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,15,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,15,0.010559999694426855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,31,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,31,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,63,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,63,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,63,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,63,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,127,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,127,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,255,0.010687999427318573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,255,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,511,0.010415999839703241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,511,0.01181866725285848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,511,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,511,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,1023,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,1023,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,1023,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,1023,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,2047,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,2047,0.012725333372751871
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,2047,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,2047,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,4095,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,4095,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,4095,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,4095,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,8191,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,8191,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,8191,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,8191,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,16383,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,16383,0.019674666225910187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,16383,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,16383,0.019530666371186573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,32767,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,32767,0.022837333381175995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,32767,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,32767,0.021717332303524017
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,65535,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,65535,0.025242666403452556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,65535,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,65535,0.02342933416366577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,float16,131071,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,float16,131071,0.0271573339899381
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,1,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,1,0.011610666910807291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,128,1,float16,fp8,131071,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,1,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,3,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,3,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,3,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,7,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,7,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,15,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,15,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,15,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,15,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,31,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,64,0,1,float16,fp8,131071,0.0269813338915507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,31,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,31,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,31,0.011749333391586939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,63,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,63,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,63,0.010725333044926325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,127,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,63,0.01219733307758967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,127,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,127,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,255,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,255,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,255,0.011653333902359009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,255,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,511,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,511,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,511,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,511,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,1023,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,1023,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,1023,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,1023,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,2047,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,2047,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,2047,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,2047,0.012709333250919977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,4095,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,4095,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,4095,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,4095,0.013434667140245438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,8191,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,8191,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,8191,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,8191,0.017338667064905167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,16383,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,16383,0.017701332767804463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,16383,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,16383,0.017263999829689663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,32767,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,32767,0.019424000134070713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,32767,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,32767,0.019050666441520054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,65535,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,65535,0.021216000119845074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,65535,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,65535,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,float16,131071,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,128,1,float16,fp8,131071,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,float16,131071,0.027258666853109997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,64,0,1,float16,fp8,131071,0.027269333600997925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,1,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,1,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,3,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,3,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,3,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,7,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,7,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,7,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,7,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,15,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,15,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,15,0.010586666564146677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,31,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,31,0.010549332946538925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,63,0.010527999450763067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,31,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,63,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,127,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,127,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,127,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,127,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,255,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,255,0.010527999450763067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,255,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,255,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,511,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,511,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,511,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,511,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,1023,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,1023,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,1023,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,2047,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,2047,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,2047,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,2047,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,4095,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,4095,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,4095,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,4095,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,8191,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,8191,0.016719999412695568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,8191,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,8191,0.016037333756685257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,16383,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,16383,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,16383,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,16383,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,32767,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,32767,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,32767,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,32767,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,65535,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,65535,0.023397333920001984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,65535,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,65535,0.023290666441122692
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,float16,131071,0.013381333400805792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,1,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,float16,131071,0.03846399982770284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,1,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,1,0.01180800050497055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,3,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,3,0.010586666564146677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,0,1,float16,fp8,131071,0.031370667119820915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,64,128,1,float16,fp8,131071,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,3,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,7,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,7,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,15,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,7,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,15,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,15,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,31,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,31,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,31,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,63,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,63,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,63,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,63,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,127,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,127,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,127,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,127,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,255,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,255,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,255,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,255,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,511,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,511,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,511,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,511,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,1023,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,1023,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,1023,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,1023,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,2047,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,2047,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,2047,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,2047,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,4095,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,4095,0.011968000481526056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,4095,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,4095,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,8191,0.011802667131026586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,8191,0.016986666868130367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,8191,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,8191,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,16383,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,16383,0.012714666624863943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,16383,0.021040000021457672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,16383,0.023498666783173878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,32767,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,32767,0.02380799998839696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,32767,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,32767,0.023210667073726654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,65535,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,65535,0.025786665578683216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,65535,0.012666666259368261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,65535,0.023578666150569916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,float16,131071,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,float16,131071,0.029509333272775013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,1,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,1,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,1,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,1,0.01055466632048289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,128,1,float16,fp8,131071,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,3,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,3,0.010565333068370819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,7,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,3,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,7,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,7,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,15,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,15,0.011834666132926941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,31,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,15,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,31,0.011839999506870905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,64,0,1,float16,fp8,131071,0.029509333272775013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,63,0.01166933278242747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,63,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,63,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,127,0.011946666985750198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,127,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,127,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,255,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,255,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,511,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,511,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,511,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,1023,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,1023,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,1023,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,1023,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,2047,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,2047,0.01267733300725619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,2047,0.012053333222866058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,2047,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,4095,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,4095,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,4095,0.012693333129088083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,4095,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,8191,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,8191,0.01692266638080279
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,8191,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,8191,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,16383,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,16383,0.019333332777023315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,16383,0.01458666721979777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,16383,0.019050666441520054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,32767,0.021375998854637146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,32767,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,32767,0.020165332903464634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,32767,0.012725333372751871
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,65535,0.01267733300725619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,65535,0.023455999791622162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,65535,0.02146666745344798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,65535,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,float16,131071,0.029253333806991577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,float16,131071,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,128,1,float16,fp8,131071,0.014789332946141561
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,1,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,1,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,1,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,1,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,3,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,3,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,3,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,7,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,7,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,15,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,15,0.010543999572594961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,15,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,15,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,31,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,31,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,31,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,31,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,63,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,63,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,64,0,1,float16,fp8,131071,0.02740799884001414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,63,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,127,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,127,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,127,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,255,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,127,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,255,0.010656000425418219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,511,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,511,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,511,0.011610666910807291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,511,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,1023,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,1023,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,1023,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,1023,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,2047,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,2047,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,2047,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,2047,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,4095,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,4095,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,4095,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,4095,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,8191,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,8191,0.016127999871969223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,8191,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,8191,0.015813333292802174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,16383,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,16383,0.018863999595244724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,16383,0.012736000120639801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,16383,0.017407999684413273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,32767,0.012533333152532578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,32767,0.021007999777793884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,32767,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,32767,0.019679999599854153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,65535,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,65535,0.025722667574882507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,65535,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,65535,0.023247999449570973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,float16,131071,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,1,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,1,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,128,1,float16,fp8,131071,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,float16,131071,0.03963200002908707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,64,0,1,float16,fp8,131071,0.03180266668399175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,1,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,3,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,3,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,7,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,7,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,7,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,7,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,15,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,15,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,15,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,15,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,31,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,31,0.010656000425418219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,31,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,31,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,63,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,63,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,127,0.010549332946538925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,127,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,127,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,127,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,255,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,255,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,255,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,511,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,511,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,511,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,511,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,1023,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,1023,0.011600000162919363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,1023,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,1023,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,2047,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,2047,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,2047,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,2047,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,4095,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,4095,0.015279999623696009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,4095,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,4095,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,8191,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,8191,0.017498667041460674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,8191,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,8191,0.016565332810084026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,16383,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,16383,0.021344001094500225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,16383,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,16383,0.01952533299724261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,32767,0.011658667276302973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,32767,0.023354666928450268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,32767,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,32767,0.023472001155217487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,65535,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,65535,0.035887998839219414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,65535,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,65535,0.030207999050617218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,float16,131071,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,float16,131071,0.05727999905745188
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,128,1,float16,fp8,131071,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,1,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,1,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,1,0.01240533341964086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,1,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,3,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,3,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,3,0.012165332833925882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,3,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,7,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,7,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,7,0.012458667159080505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,15,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,15,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,64,0,1,float16,fp8,131071,0.05040533343950907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,15,0.012319999436537424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,31,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,31,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,31,0.011610666910807291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,31,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,63,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,63,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,63,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,127,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,127,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,127,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,255,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,255,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,255,0.011621333658695221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,511,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,511,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,511,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,511,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,1023,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,1023,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,1023,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,1023,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,2047,0.014186666657527288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,2047,0.017018667111794155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,2047,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,2047,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,4095,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,4095,0.02024000013868014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,4095,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,4095,0.01926933353145917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,8191,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,8191,0.02701866626739502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,8191,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,8191,0.025429333249727886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,float16,16383,0.014805333067973455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,128,1,float16,fp8,16383,0.014746667196353277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,fp8,16383,0.0359199990828832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,64,0,1,float16,float16,16383,0.041946664452552795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,1,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,1,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,3,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,3,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,3,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,3,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,7,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,7,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,7,0.011792000383138657
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,15,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,15,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,31,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,31,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,31,0.01210133358836174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,31,0.01201066623131434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,63,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,63,0.012191999703645706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,127,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,127,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,127,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,127,0.01219733307758967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,255,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,255,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,255,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,255,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,511,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,511,0.011653333902359009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,511,0.012335999558369318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,511,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,1023,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,1023,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,1023,0.011578666667143503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,1023,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,2047,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,2047,0.01481066644191742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,2047,0.01933866615096728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,2047,0.017808000246683758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,4095,0.014794666320085526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,4095,0.023376000424226124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,4095,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,4095,0.02348266790310542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,8191,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,8191,0.03845333307981491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,8191,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,8191,0.03193599979082743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,float16,16383,0.05884799857934316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,float16,16383,0.014384000251690546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,1,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,128,1,float16,fp8,16383,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,1,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,1,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,1,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,64,0,1,float16,fp8,16383,0.05219733218352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,3,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,3,0.01441066712141037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,3,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,7,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,3,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,7,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,7,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,7,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,15,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,15,0.01191466674208641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,15,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,15,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,31,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,31,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,31,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,31,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,63,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,63,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,63,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,63,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,127,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,127,0.011834666132926941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,127,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,127,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,255,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,255,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,255,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,255,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,511,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,511,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,511,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,511,0.01470400020480156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,1023,0.012714666624863943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,1023,0.016879999389251072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,1023,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,1023,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,2047,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,2047,0.025120000044504803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,2047,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,2047,0.023215999205907185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,4095,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,4095,0.015392000476519266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,4095,0.03994666785001755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,4095,0.031343999008337654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,8191,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,8191,0.01534933348496755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,8191,0.06201600035031637
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,8191,0.052655999859174095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,float16,16383,0.015589332828919092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,float16,16383,0.10378666718800862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,1,0.01591466615597407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,128,1,float16,fp8,16383,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,1,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,1,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,3,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,1,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,64,0,1,float16,fp8,16383,0.08743466933568318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,3,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,3,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,7,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,3,0.01402666668097178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,7,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,7,0.014405333747466406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,7,0.014618666221698126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,15,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,15,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,15,0.013909333695967993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,15,0.014208000153303146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,31,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,31,0.013525333255529404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,31,0.01394133393963178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,31,0.014730667074521383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,63,0.013610667238632837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,63,0.01339200014869372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,63,0.014538666854302088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,63,0.01451733335852623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,127,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,127,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,127,0.014298666268587112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,127,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,255,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,255,0.014698666830857595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,255,0.014773332824309668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,255,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,511,0.014837333311637243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,511,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,511,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,511,0.01699200024207433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,1023,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,1023,0.023498666783173878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,1023,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,1023,0.021562665700912476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,2047,0.01754133279124896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,2047,0.03977599988381068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,2047,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,2047,0.032058666149775185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,4095,0.01748266691962878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,4095,0.017263999829689663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,4095,0.06186666587988535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,4095,0.05358933409055074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,8191,0.01762666677435239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,8191,0.10352533062299092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,8191,0.017583999782800674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,8191,0.08770666519800822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,fp8,16383,0.0174346665541331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,128,1,float16,float16,16383,0.017968000223239262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,float16,16383,0.1877066691716512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,1,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,1,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,1,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,1,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,3,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,3,0.01166933278242747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,3,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,64,0,1,float16,fp8,16383,0.15682666500409445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,7,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,7,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,7,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,15,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,15,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,15,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,31,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,31,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,31,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,31,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,63,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,63,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,63,0.011882666498422623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,63,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,127,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,127,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,127,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,127,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,255,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,255,0.010725333044926325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,255,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,255,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,511,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,511,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,511,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,511,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,1023,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,1023,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,1023,0.012736000120639801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,1023,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,2047,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,2047,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,2047,0.013408000270525614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,2047,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,4095,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,4095,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,4095,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,4095,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,8191,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,8191,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,8191,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,8191,0.018357332795858383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,16383,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,16383,0.019776000330845516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,16383,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,16383,0.021162666380405426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,32767,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,32767,0.02143466720978419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,32767,0.01357866699496905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,32767,0.021429332594076794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,65535,0.013408000270525614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,65535,0.025034666061401367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,65535,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,65535,0.023733332753181458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,float16,131071,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,float16,131071,0.03137599925200144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,1,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,1,0.012037333101034164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,1,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,1,0.012346666306257248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,3,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,128,1,float16,fp8,131071,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,3,0.011877333124478659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,3,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,3,0.011653333902359009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,7,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,7,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,7,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,7,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,15,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,15,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,15,0.011760000139474869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,15,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,31,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,31,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,31,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,63,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,63,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,63,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,64,0,1,float16,fp8,131071,0.030762667457262676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,127,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,127,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,63,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,255,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,255,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,255,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,511,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,255,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,511,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,511,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,511,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,1023,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,1023,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,1023,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,1023,0.011706666400035223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,2047,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,2047,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,2047,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,2047,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,4095,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,4095,0.015471999843915304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,4095,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,4095,0.017375999440749485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,8191,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,8191,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,8191,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,8191,0.01823466643691063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,16383,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,16383,0.019509332875410717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,16383,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,16383,0.019130667050679524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,32767,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,32767,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,32767,0.021477334201335907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,32767,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,65535,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,65535,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,65535,0.02513066679239273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,65535,0.024911999702453613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,float16,131071,0.014762666076421738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,fp8,131071,0.03391999999682108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,128,1,float16,fp8,131071,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,64,0,1,float16,float16,131071,0.04238933324813843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,1,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,1,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,3,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,3,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,3,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,3,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,7,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,7,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,7,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,15,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,15,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,15,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,31,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,31,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,31,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,31,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,63,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,63,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,63,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,63,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,127,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,127,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,127,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,255,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,255,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,255,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,255,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,511,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,511,0.013781332721312841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,511,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,511,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,1023,0.011663999408483505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,1023,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,1023,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,2047,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,2047,0.015530666957298914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,2047,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,2047,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,4095,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,4095,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,4095,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,4095,0.017045332739750545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,8191,0.013530666629473368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,8191,0.019226666539907455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,8191,0.01718933383623759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,8191,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,16383,0.013728000223636627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,16383,0.021541332205136616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,16383,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,16383,0.02075733368595441
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,32767,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,32767,0.025285333395004272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,32767,0.013434667140245438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,32767,0.023823998868465424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,65535,0.013754667093356451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,65535,0.038575999438762665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,65535,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,65535,0.03151999910672506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,float16,131071,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,float16,131071,0.05991466840108236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,1,0.01166933278242747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,1,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,1,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,1,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,3,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,3,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,128,1,float16,fp8,131071,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,3,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,7,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,3,0.012053333222866058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,7,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,64,0,1,float16,fp8,131071,0.052746668457984924
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,15,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,15,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,15,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,15,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,31,0.010496000448862711
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,31,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,31,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,31,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,63,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,63,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,63,0.0116799995303154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,127,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,127,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,127,0.011594666788975397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,127,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,255,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,255,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,255,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,255,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,511,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,511,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,511,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,511,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,1023,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,1023,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,1023,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,1023,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,2047,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,2047,0.015589332828919092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,2047,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,2047,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,4095,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,4095,0.016970666746298473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,4095,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,4095,0.015573333948850632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,8191,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,8191,0.019237333287795384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,8191,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,8191,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,16383,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,16383,0.023797333240509033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,16383,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,16383,0.023557332654794056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,32767,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,32767,0.037717332442601524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,32767,0.013557333499193192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,32767,0.031317333380381264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,65535,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,65535,0.05842133363087972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,65535,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,65535,0.052058666944503784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,float16,131071,0.015477333217859268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,float16,131071,0.10317333539326985
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,1,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,1,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,128,1,float16,fp8,131071,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,1,0.012186666329701742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,3,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,1,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,3,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,3,0.012752000242471695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,3,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,7,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,7,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,7,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,7,0.0136266661187013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,15,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,15,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,64,0,1,float16,fp8,131071,0.08881066242853801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,15,0.01268799975514412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,15,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,31,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,31,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,31,0.012671999633312225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,63,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,31,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,63,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,63,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,63,0.011802667131026586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,127,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,127,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,127,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,255,0.012629333883523941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,255,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,511,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,511,0.012618667135636011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,511,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,511,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,1023,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,1023,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,1023,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,1023,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,2047,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,2047,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,2047,0.019189332922299702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,2047,0.019973333925008774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,4095,0.014922666052977243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,4095,0.01488000030318896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,4095,0.027141332626342773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,4095,0.0252960001428922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,float16,8191,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,128,1,float16,fp8,8191,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,float16,8191,0.0435146689414978
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,64,0,1,float16,fp8,8191,0.03570666660865148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,1,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,1,0.012175999581813812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,1,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,1,0.012709333250919977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,3,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,3,0.012645332763592402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,3,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,3,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,7,0.012624000509579977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,7,0.012362666428089142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,7,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,7,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,15,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,15,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,15,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,15,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,31,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,31,0.012026666353146235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,31,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,63,0.012149333953857422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,31,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,63,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,63,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,63,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,127,0.012624000509579977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,127,0.011978667229413986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,127,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,127,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,255,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,255,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,255,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,255,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,511,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,511,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,511,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,511,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,1023,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,1023,0.01695999999841054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,1023,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,1023,0.015482666591803232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,2047,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,2047,0.02383466561635335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,2047,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,2047,0.023472001155217487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,4095,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,4095,0.041509332756201424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,4095,0.015498666713635126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,4095,0.03161066770553589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,float16,8191,0.016149333367745083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,float16,8191,0.06170133252938589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,128,1,float16,fp8,8191,0.01532799998919169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,1,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,1,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,1,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,64,0,1,float16,fp8,8191,0.05328000088532766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,1,0.014159999787807465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,3,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,3,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,3,0.014549333602190018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,3,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,7,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,7,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,7,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,15,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,15,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,15,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,7,0.01579733317097028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,31,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,15,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,31,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,31,0.01452800010641416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,63,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,63,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,31,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,63,0.014890667051076889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,63,0.014885333677132925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,127,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,127,0.013983999689420065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,127,0.014922666052977243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,127,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,255,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,255,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,255,0.014645333091417948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,255,0.013637332866589228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,511,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,511,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,511,0.014837333311637243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,1023,0.014746667196353277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,511,0.017632000148296356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,1023,0.02311466634273529
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,1023,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,1023,0.021349333226680756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,2047,0.017594666530688603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,2047,0.016938666502634685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,2047,0.039834665755430855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,2047,0.033557333052158356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,4095,0.01746133342385292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,4095,0.017488000293572743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,4095,0.06216000020503998
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,4095,0.0535093347231547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,float16,8191,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,128,1,float16,fp8,8191,0.017077332983414333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,float16,8191,0.10353599985440572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,64,0,1,float16,fp8,8191,0.08716266353925069
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,1,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,1,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,1,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,1,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,3,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,3,0.017338667064905167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,3,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,3,0.01746133342385292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,7,0.017450666675964992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,7,0.017184000462293625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,7,0.01764800027012825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,7,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,15,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,15,0.01759999990463257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,15,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,15,0.01720533271630605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,31,0.017583999782800674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,31,0.017637333522240322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,31,0.01714666684468587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,31,0.01732800031701724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,63,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,63,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,63,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,63,0.017290666699409485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,127,0.016938666502634685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,127,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,127,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,127,0.016949333250522614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,255,0.01801066721479098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,255,0.01727466657757759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,255,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,255,0.01714666684468587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,511,0.01904533306757609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,511,0.017173333714405697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,511,0.0232640008131663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,511,0.021338666478792827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,1023,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,1023,0.03533866753180822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,1023,0.018181333939234417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,1023,0.029461334149042766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,2047,0.021216000119845074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,2047,0.057328000664711
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,2047,0.02128000060717265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,2047,0.05165866514046987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,4095,0.021386665602525074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,4095,0.09503466884295146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,4095,0.021477334201335907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,4095,0.0810346653064092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,float16,8191,0.021941334009170532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,float16,8191,0.16869332393010458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,128,1,float16,fp8,8191,0.021104000508785248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,1,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,1,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,1,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,64,0,1,float16,fp8,8191,0.14060800274213156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,1,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,3,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,3,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,3,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,7,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,7,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,7,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,3,0.013872000078360239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,7,0.013594667116800943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,15,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,15,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,15,0.0136266661187013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,15,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,31,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,31,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,31,0.013658666362365087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,31,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,63,0.01349866638580958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,63,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,63,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,63,0.013690666606028875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,127,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,127,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,127,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,127,0.01379199946920077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,255,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,255,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,255,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,255,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,511,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,511,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,511,0.01368533323208491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,511,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,1023,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,1023,0.017594666530688603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,1023,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,1023,0.017573333034912746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,float16,2047,0.01728533332546552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,float16,2047,0.027600000301996868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,128,1,float16,fp8,2047,0.018895999838908512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,1,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,1,0.014885333677132925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,64,0,1,float16,fp8,2047,0.02515733242034912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,1,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,1,0.014773332824309668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,3,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,3,0.014426667243242264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,3,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,7,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,3,0.014661333213249842
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,7,0.013754667093356451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,7,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,7,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,15,0.014576000471909841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,15,0.014032000054915747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,15,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,15,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,31,0.013610667238632837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,31,0.015573333948850632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,31,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,31,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,63,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,63,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,63,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,127,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,63,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,127,0.015685333559910457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,127,0.014752000570297241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,127,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,255,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,255,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,255,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,255,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,511,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,511,0.017765333255132038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,511,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,511,0.017653333644072216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,1023,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,1023,0.021226666867733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,1023,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,1023,0.022266666094462078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,float16,2047,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,float16,2047,0.04142933338880539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,128,1,float16,fp8,2047,0.017386666188637417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,64,0,1,float16,fp8,2047,0.03335466732581457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,1,0.017658667018016178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,1,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,1,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,1,0.01786133274435997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,3,0.01728533332546552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,3,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,3,0.017711999515692394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,3,0.017653333644072216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,7,0.017605333278576534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,7,0.01720000058412552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,7,0.01758933315674464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,7,0.01752000053723653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,15,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,15,0.018005333840847015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,15,0.017738666385412216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,15,0.01756799966096878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,31,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,31,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,31,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,31,0.01830400029818217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,63,0.0174346665541331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,63,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,63,0.01894933357834816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,63,0.01782400036851565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,127,0.017349333812793095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,127,0.01757866640885671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,127,0.018042666216691334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,127,0.01788266624013583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,255,0.01887999971707662
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,255,0.01732800031701724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,255,0.01869333287080129
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,255,0.017562666287024815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,511,0.01945066700379054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,511,0.02333866556485494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,511,0.01903466631968816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,511,0.022064000368118286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,1023,0.017456000049908955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,1023,0.03586133321126302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,1023,0.017530667285124462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,1023,0.029637334247430164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,fp8,2047,0.020992000897725422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,128,1,float16,float16,2047,0.021322667598724365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,float16,2047,0.058464000622431435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,1,0.025237334271272022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,1,0.02350933353106181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,1,0.025397333006064098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,64,0,1,float16,fp8,2047,0.05050666630268097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,1,0.023573334018389385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,3,0.024869332710901897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,3,0.025263999899228413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,3,0.025301332275072735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,3,0.02514133354028066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,7,0.02515733242034912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,7,0.025066666305065155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,7,0.02516266703605652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,7,0.025258667767047882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,15,0.02516799916823705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,15,0.02508266766866048
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,15,0.025237334271272022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,15,0.02510933329661687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,31,0.0249493345618248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,31,0.025055999557177227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,31,0.02587733417749405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,31,0.02513066679239273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,63,0.02508266766866048
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,63,0.025381334125995636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,63,0.025333332518736523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,63,0.024986666937669117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,127,0.02515733242034912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,127,0.023402666052182514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,127,0.02535466601451238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,127,0.0252960001428922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,255,0.025407999753952026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,255,0.025973332424958546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,255,0.025546667476495106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,255,0.025072000920772552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,511,0.025461333493391674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,511,0.03772799919048945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,511,0.025034666061401367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,511,0.03233599911133448
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,1023,0.02555199960867564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,1023,0.05599466462930044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,1023,0.025536000728607178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,1023,0.05004266897837321
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,float16,2047,0.02956799914439519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,float16,2047,0.09771733482678731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,128,1,float16,fp8,2047,0.027434666951497395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,1,0.01595199977358182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,64,0,1,float16,fp8,2047,0.08531733353932698
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,1,0.015114666273196539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,1,0.015647999942302704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,1,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,3,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,3,0.015610666324694952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,3,0.01534933348496755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,3,0.015696000307798386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,7,0.01534933348496755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,7,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,7,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,7,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,15,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,15,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,15,0.01573866605758667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,15,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,31,0.015381333728631338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,31,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,31,0.015557333827018738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,31,0.016293333222468693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,63,0.015450666348139444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,63,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,63,0.01551466683546702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,127,0.014896000425020853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,63,0.01669866715868314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,127,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,127,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,127,0.015429332852363586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,255,0.015824000040690105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,255,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,255,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,255,0.01552533358335495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,511,0.01552533358335495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,511,0.019434666881958645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,511,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,511,0.01947733387351036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,float16,1023,0.015520000209410986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,128,1,float16,fp8,1023,0.015487999965747198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,float16,1023,0.025616000096003216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,64,0,1,float16,fp8,1023,0.02510400116443634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,1,0.017722666263580322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,1,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,1,0.019391999890406925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,1,0.01924266666173935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,3,0.0176959993938605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,3,0.017583999782800674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,3,0.019317333896954853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,3,0.018986667195955913
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,7,0.017136000096797943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,7,0.01754666616519292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,7,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,7,0.01842133328318596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,15,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,15,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,15,0.019424000134070713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,15,0.017797333498795826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,31,0.01793066660563151
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,31,0.017903999735911686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,31,0.01903466631968816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,31,0.017407999684413273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,63,0.019215999792019527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,63,0.0189280000825723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,63,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,63,0.01752000053723653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,127,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,127,0.018816000471512478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,127,0.018901333212852478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,127,0.01871466636657715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,255,0.01869333287080129
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,255,0.019285333653291065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,255,0.018687999496857326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,255,0.02085866779088974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,511,0.019167999426523846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,511,0.023386667172114056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,511,0.018981333822011948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,511,0.0235359991590182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,float16,1023,0.01860800012946129
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,float16,1023,0.035530666510264076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,128,1,float16,fp8,1023,0.019007999449968338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,64,0,1,float16,fp8,1023,0.030960001051425934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,1,0.02521066615978877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,1,0.025797332326571148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,1,0.025439999997615814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,1,0.025445332129796345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,3,0.02584533393383026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,3,0.025514667232831318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,3,0.02536533276240031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,7,0.025450666745503742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,3,0.025781333446502686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,7,0.025040000677108765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,7,0.025237334271272022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,7,0.02589866767326991
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,15,0.02502399931351344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,15,0.02613866577545802
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,15,0.025834667185942333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,15,0.025349333882331848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,31,0.025311999022960663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,31,0.025455998877684276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,31,0.02536533276240031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,31,0.02553066611289978
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,63,0.025813333690166473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,63,0.025445332129796345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,63,0.025653332471847534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,63,0.02531733363866806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,127,0.025434667865435284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,127,0.025445332129796345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,127,0.025258667767047882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,127,0.02510933329661687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,255,0.025487999121348064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,255,0.025434667865435284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,255,0.025407999753952026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,255,0.025429333249727886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,511,0.02775466690460841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,511,0.03764266769091288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,511,0.025487999121348064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,511,0.03443199892838796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,float16,1023,0.056186666091283165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,fp8,1023,0.025205334027608235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,128,1,float16,float16,1023,0.02703999976317088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,64,0,1,float16,fp8,1023,0.04986133178075155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,1,0.03949866692225138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,1,0.03772266705830892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,1,0.03769599894682566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,1,0.03984000037113825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,3,0.03939733405907949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,3,0.03941866755485535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,3,0.03799466788768768
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,7,0.040031999349594116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,3,0.0376800000667572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,7,0.03953066716591517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,7,0.03782933453718821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,15,0.03987200061480204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,7,0.03789866715669632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,15,0.03935466706752777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,15,0.037989333271980286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,15,0.03754133234421412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,31,0.039861333866914116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,31,0.03976533313592275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,31,0.037621334195137024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,31,0.03766933331886927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,63,0.03947199881076813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,63,0.039594667653242745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,63,0.03755733370780945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,63,0.03766933331886927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,127,0.039887999494870506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,127,0.03766400118668874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,127,0.03766933331886927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,127,0.03775466730197271
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,255,0.04162666698296865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,255,0.041706666350364685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,255,0.03939199944337209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,255,0.037461332976818085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,511,0.04190400242805481
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,511,0.03967999915281931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,511,0.06037333110968272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,511,0.056277334690093994
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,float16,1023,0.042037333051363625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,1,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,128,1,float16,fp8,1023,0.03944533318281174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,float16,1023,0.09563733140627544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,1,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,64,0,1,float16,fp8,1023,0.08504000306129456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,1,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,1,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,3,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,3,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,3,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,3,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,7,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,7,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,7,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,7,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,15,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,15,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,31,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,31,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,31,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,63,0.010591999938090643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,63,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,63,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,127,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,127,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,127,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,255,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,255,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,255,0.012378666549921036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,255,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,511,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,511,0.012138667205969492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,511,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,511,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,1023,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,1023,0.012576000144084295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,1023,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,1023,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,2047,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,2047,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,2047,0.013797332843144735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,2047,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,4095,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,4095,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,4095,0.01704000060757001
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,4095,0.01714666684468587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,8191,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,8191,0.017594666530688603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,8191,0.014090667168299357
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,8191,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,16383,0.013365333278973898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,16383,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,16383,0.019914666811625164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,16383,0.01947733387351036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,32767,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,32767,0.02349333216746648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,32767,0.013776000589132309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,32767,0.021407999098300934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,65535,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,65535,0.02779199928045273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,65535,0.013722666849692663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,65535,0.02606400102376938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,float16,131071,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,float16,131071,0.04450666904449463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,1,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,1,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,1,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,3,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,3,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,128,1,float16,fp8,131071,0.017045332739750545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,3,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,7,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,15,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,15,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,15,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,15,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,31,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,31,0.012005332857370377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,31,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,31,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,63,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,63,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,63,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,64,0,1,float16,fp8,131071,0.03576533248027166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,63,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,127,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,127,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,127,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,255,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,255,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,511,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,511,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,511,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,511,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,1023,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,1023,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,1023,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,1023,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,2047,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,2047,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,2047,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,2047,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,4095,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,4095,0.01700266698996226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,4095,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,4095,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,8191,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,8191,0.01773333301146825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,8191,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,8191,0.017194667210181553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,16383,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,16383,0.019215999792019527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,16383,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,16383,0.01947733387351036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,32767,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,32767,0.025434667865435284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,32767,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,32767,0.023989332218964893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,65535,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,65535,0.03930133332808813
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,65535,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,65535,0.031370667119820915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,float16,131071,0.014842666685581207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,float16,131071,0.06006933252016703
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,1,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,128,1,float16,fp8,131071,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,1,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,1,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,3,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,3,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,3,0.012175999581813812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,7,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,7,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,7,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,7,0.012069333344697952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,15,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,15,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,15,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,15,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,31,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,31,0.011973333855470022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,31,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,63,0.01184533288081487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,63,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,63,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,63,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,127,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,64,0,1,float16,fp8,131071,0.05212266743183136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,127,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,127,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,127,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,255,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,255,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,255,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,511,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,511,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,511,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,511,0.013823999712864557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,1023,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,1023,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,1023,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,1023,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,2047,0.01258133351802826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,2047,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,2047,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,2047,0.015360000232855478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,4095,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,4095,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,4095,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,4095,0.01621333385507266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,8191,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,8191,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,8191,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,8191,0.019109333554903667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,16383,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,16383,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,16383,0.024319998919963837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,16383,0.023365333676338196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,32767,0.013749333719412485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,32767,0.03761066744724909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,32767,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,32767,0.031632001201311745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,65535,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,65535,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,65535,0.058277333776156105
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,65535,0.0521919975678126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,float16,131071,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,128,1,float16,fp8,131071,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,float16,131071,0.10149332880973816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,64,0,1,float16,fp8,131071,0.08930133779843648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,1,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,1,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,1,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,1,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,3,0.010474666953086853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,3,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,7,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,7,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,7,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,15,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,15,0.010591999938090643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,15,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,31,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,31,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,31,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,31,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,63,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,63,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,63,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,127,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,127,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,127,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,127,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,255,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,255,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,255,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,255,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,511,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,511,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,511,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,511,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,1023,0.01163200040658315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,1023,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,1023,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,1023,0.013429333766301474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,2047,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,2047,0.01695466662446658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,2047,0.01350933313369751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,2047,0.015487999965747198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,4095,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,4095,0.019178666174411774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,4095,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,4095,0.01922133316596349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,8191,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,8191,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,8191,0.023152001202106476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,8191,0.02189333240191142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,16383,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,16383,0.037978666524092354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,16383,0.03126399964094162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,16383,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,32767,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,32767,0.05845866600672404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,32767,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,32767,0.0521919975678126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,65535,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,65535,0.10012267033259074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,65535,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,65535,0.08708799878756206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,float16,131071,0.01533866673707962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,float16,131071,0.1843573252360026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,128,1,float16,fp8,131071,0.015482666591803232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,1,0.020021333048741024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,1,0.020773333807786305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,1,0.021183999876181286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,64,0,1,float16,fp8,131071,0.15732799967130026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,1,0.02073066681623459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,3,0.019237333287795384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,3,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,3,0.019386666516462963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,3,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,7,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,7,0.019296000401178997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,7,0.019354666272799175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,7,0.021498667697111767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,15,0.01958400011062622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,15,0.019610666980346043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,15,0.01959466685851415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,15,0.021333334346612293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,31,0.019600000232458115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,31,0.019648000597953796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,31,0.019658666104078293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,31,0.02128533273935318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,63,0.01972266659140587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,63,0.019343999524911244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,63,0.019920000185569126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,63,0.0195573332409064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,127,0.019130667050679524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,127,0.01926400015751521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,127,0.019391999890406925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,127,0.019861333072185516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,255,0.020986666282018025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,255,0.01945066700379054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,255,0.02123733361562093
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,255,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,float16,511,0.021205333371957142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,float16,511,0.025663999219735462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,128,1,float16,fp8,511,0.021456000705560047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,64,0,1,float16,fp8,511,0.025557334224383037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,1,0.025205334027608235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,1,0.025663999219735462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,1,0.025850666066010792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,1,0.025786665578683216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,3,0.025472000241279602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,3,0.02550933261712392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,3,0.025573333104451496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,3,0.025445332129796345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,7,0.026165333886941273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,7,0.025727999707063038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,7,0.025349333882331848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,7,0.02536533276240031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,15,0.025594666600227356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,15,0.025466665625572205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,15,0.025386666258176167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,15,0.025466665625572205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,31,0.025653332471847534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,31,0.026101333399613697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,31,0.026250667870044708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,63,0.02677333354949951
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,31,0.025674665967623394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,63,0.026261332134405773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,63,0.02537599951028824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,63,0.02603733291228612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,127,0.02587733417749405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,127,0.025781333446502686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,127,0.025562666356563568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,127,0.02595199892918269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,255,0.027327999472618103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,255,0.025546667476495106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,255,0.025834667185942333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,255,0.025813333690166473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,float16,511,0.027445333699385326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,float16,511,0.037733333806196846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,0,1,float16,fp8,511,0.03364799916744232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,1,0.039936001102129616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,64,128,1,float16,fp8,511,0.027285332481066387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,1,0.03974399964014689
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,1,0.03939199944337209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,1,0.0395359992980957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,3,0.03963200002908707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,3,0.0400693342089653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,3,0.03956266740957896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,3,0.03969600051641464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,7,0.039733332892258964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,7,0.03990933299064636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,7,0.03969600051641464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,7,0.03961066653331121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,15,0.03972800076007843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,15,0.040021332601706185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,15,0.03939199944337209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,15,0.03951466580231985
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,31,0.039887999494870506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,31,0.03985599925120672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,31,0.03900266687075297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,31,0.0395359992980957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,63,0.03977066775163015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,63,0.03977066775163015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,63,0.03958400090535482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,63,0.0395359992980957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,127,0.03977066775163015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,127,0.039850667119026184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,127,0.03984533250331879
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,127,0.0399893323580424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,255,0.04187199970086416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,255,0.04205866654713949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,255,0.04001066585381826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,255,0.03811733424663544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,float16,511,0.04181866844495138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,float16,511,0.06015466650327047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,128,1,float16,fp8,511,0.03977066775163015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,1,0.06837333242098491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,64,0,1,float16,fp8,511,0.05676800012588501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,1,0.06865600248177846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,1,0.0662666658560435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,1,0.06496533254782359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,3,0.06677333513895671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,3,0.06454400221506755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,3,0.06834133466084798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,3,0.06460266808668773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,7,0.06715199848016103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,7,0.06842666864395142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,7,0.0662666658560435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,7,0.06448533137639363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,15,0.06874133149782817
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,15,0.06849066913127899
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,15,0.06644266843795776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,15,0.06618666648864746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,31,0.06885866820812225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,31,0.06846400101979573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,31,0.06482666730880737
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,31,0.06618133187294006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,63,0.06861866513888042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,63,0.06825066606203715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,63,0.06486933430035909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,63,0.06629866858323415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,127,0.07052266597747803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,127,0.0705813318490982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,127,0.06654933094978333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,127,0.06635733445485432
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,255,0.0732586681842804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,255,0.0717386653025945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,255,0.06659733255704244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,255,0.06890133519967397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,float16,511,0.07449600100517273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,float16,511,0.1077280044555664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,128,1,float16,fp8,511,0.06694933275381725
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,64,0,1,float16,fp8,511,0.09947733084360759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,1,0.03032533327738444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,1,0.029616000751654308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,1,0.030239999294281006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,1,0.03194666653871536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,3,0.029829333225886028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,3,0.02942399928967158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,3,0.030063999195893604
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,7,0.029845332105954487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,3,0.03052799900372823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,7,0.030266667405764263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,7,0.03143999973932902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,7,0.03057066599527995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,15,0.03009066730737686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,15,0.02980799973011017
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,15,0.03054933249950409
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,15,0.032511999209721885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,31,0.02934933453798294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,31,0.03018666555484136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,31,0.03182933231194814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,63,0.029520000020662945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,31,0.03271466741959254
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,63,0.03068800022204717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,63,0.03173333406448364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,63,0.029605334003766377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,127,0.03027733415365219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,127,0.030293333033720653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,127,0.03177600105603536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,127,0.029535998900731403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,float16,255,0.031957333286603294
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,float16,255,0.02961066613594691
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,128,1,float16,fp8,255,0.03244800120592117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,64,0,1,float16,fp8,255,0.029637334247430164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,1,0.04211199780305227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,1,0.041984001795450844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,1,0.04190400242805481
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,1,0.041509332756201424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,3,0.04172799984614054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,3,0.04186133543650309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,3,0.041877334316571556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,3,0.04154666761557261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,7,0.04194133480389913
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,7,0.04162666698296865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,7,0.041637333730856575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,7,0.042064001162846885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,15,0.041536000867684685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,15,0.041797334949175514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,15,0.04200533529122671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,15,0.04145599901676178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,31,0.04201599955558777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,31,0.04212800165017446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,31,0.041738669077555336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,63,0.04200533529122671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,63,0.04188266893227895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,63,0.04224533339341482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,63,0.04201066493988037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,127,0.041834667325019836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,127,0.042090664307276406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,127,0.04191466669241587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,float16,255,0.04468800127506256
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,31,0.04329599936803182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,float16,255,0.04609066744645437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,128,1,float16,fp8,255,0.041834667325019836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,1,0.07098133365313213
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,255,0.042261332273483276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,1,0.06904000043869019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,1,0.06862399975458781
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,64,0,1,float16,fp8,127,0.04211199780305227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,3,0.07100266714890797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,3,0.0713866651058197
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,3,0.0684799998998642
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,3,0.0695360004901886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,7,0.07070399820804596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,7,0.07102400064468384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,7,0.06890133519967397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,7,0.06878399848937988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,1,0.07160533467928569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,15,0.07014933228492737
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,15,0.07040533423423767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,15,0.06860800087451935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,15,0.06956266860167186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,31,0.07039466500282288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,31,0.07041599849859874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,31,0.0684853345155716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,31,0.06881066660086314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,63,0.07054399947325389
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,63,0.07082666456699371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,63,0.06833066542943318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,63,0.0690773328145345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,127,0.07287999987602234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,127,0.0730506678422292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,127,0.06926399966080983
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,127,0.06924266616503398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,float16,255,0.07667199770609538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,float16,255,0.0749013324578603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,128,1,float16,fp8,255,0.0705813318490982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,64,0,1,float16,fp8,255,0.07213866710662842
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,1,0.12943466504414877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,1,0.1290880044301351
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,1,0.1220906674861908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,1,0.12192533413569133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,3,0.1290880044301351
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,3,0.129013329744339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,3,0.12212799986203511
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,3,0.12210133671760559
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,7,0.12898666659990946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,7,0.12204266587893169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,7,0.12128000458081563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,15,0.12902933359146118
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,15,0.12785067160924277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,15,0.12122133374214172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,15,0.12169067064921062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,31,0.12813333670298258
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,31,0.12882133324941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,31,0.12174399693806966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,7,0.13032533725102743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,63,0.12971199552218118
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,31,0.12258666753768921
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,63,0.12956800063451132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,63,0.12224533160527547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,63,0.121888001759847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,127,0.12924266854921976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,127,0.12879467010498047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,127,0.12384532888730367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,127,0.12397332986195882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,float16,255,0.13751467068990073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,1,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,1,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,1,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,float16,255,0.13236266374588013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,1,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,3,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,128,1,float16,fp8,255,0.12589866916338602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,64,0,1,float16,fp8,255,0.12436266740163167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,3,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,7,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,7,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,7,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,15,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,15,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,15,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,31,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,31,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,31,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,31,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,63,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,63,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,63,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,127,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,127,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,255,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,255,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,511,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,511,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,511,0.012655999511480331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,511,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,1023,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,1023,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,1023,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,1023,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,2047,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,2047,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,2047,0.013781332721312841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,2047,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,4095,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,4095,0.017162666966517765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,4095,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,4095,0.01379199946920077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,8191,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,8191,0.01886933296918869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,8191,0.013983999689420065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,8191,0.01918399954835574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,16383,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,16383,0.013898666948080063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,16383,0.023552000522613525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,16383,0.021327999730904896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,32767,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,32767,0.027509334186712902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,32767,0.026149332523345947
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,32767,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,float16,65535,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,128,1,float16,fp8,65535,0.013797332843144735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,float16,65535,0.04195733368396759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,64,0,1,float16,fp8,65535,0.0352960005402565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,1,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,1,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,1,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,1,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,3,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,3,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,3,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,3,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,7,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,7,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,7,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,15,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,15,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,15,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,31,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,31,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,31,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,63,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,63,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,63,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,63,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,127,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,127,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,127,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,127,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,255,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,255,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,255,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,255,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,511,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,511,0.012213333199421564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,511,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,511,0.011600000162919363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,1023,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,1023,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,1023,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,1023,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,2047,0.01369599997997284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,2047,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,2047,0.013418667018413544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,2047,0.01551466683546702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,4095,0.013573333621025085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,4095,0.017077332983414333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,4095,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,4095,0.01586666703224182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,8191,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,8191,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,8191,0.019189332922299702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,8191,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,16383,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,16383,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,16383,0.02333866556485494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,16383,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,32767,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,32767,0.03885866701602936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,32767,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,32767,0.03160533308982849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,float16,65535,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,float16,65535,0.058634668588638306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,1,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,1,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,128,1,float16,fp8,65535,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,1,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,3,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,3,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,3,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,7,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,7,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,64,0,1,float16,fp8,65535,0.052149335543314614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,7,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,15,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,31,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,31,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,31,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,31,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,63,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,63,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,63,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,127,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,127,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,127,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,255,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,255,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,255,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,255,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,511,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,511,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,511,0.012655999511480331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,511,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,1023,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,1023,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,1023,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,1023,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,2047,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,2047,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,2047,0.016805333395799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,2047,0.01695999999841054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,4095,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,4095,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,4095,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,4095,0.01903466631968816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,8191,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,8191,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,8191,0.023562667270501454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,8191,0.023589332898457844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,16383,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,16383,0.037434667348861694
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,16383,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,16383,0.03160533308982849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,32767,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,32767,0.0606879989306132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,32767,0.013408000270525614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,32767,0.052298665046691895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,float16,65535,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,float16,65535,0.09996267159779866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,128,1,float16,fp8,65535,0.013525333255529404
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,1,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,1,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,1,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,3,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,3,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,3,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,7,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,7,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,7,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,64,0,1,float16,fp8,65535,0.0867680013179779
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,7,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,15,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,15,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,15,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,31,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,31,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,31,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,63,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,63,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,63,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,127,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,127,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,63,0.012117333710193634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,127,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,255,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,255,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,255,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,511,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,255,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,511,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,511,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,511,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,1023,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,1023,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,1023,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,1023,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,2047,0.016986666868130367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,2047,0.01461333284775416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,2047,0.017535999417304993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,2047,0.013872000078360239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,4095,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,4095,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,4095,0.02327466756105423
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,4095,0.021941334009170532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,8191,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,8191,0.013605333864688873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,8191,0.037434667348861694
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,8191,0.029946667452653248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,16383,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,16383,0.01422400027513504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,16383,0.05845333139101664
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,16383,0.05100800096988678
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,32767,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,32767,0.09999466935793559
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,32767,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,32767,0.08496000369389851
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,float16,65535,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,128,1,float16,fp8,65535,0.01421333352724711
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,float16,65535,0.18281066417694092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,64,0,1,float16,fp8,65535,0.15456533432006836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,1,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,1,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,7,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,3,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,3,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,7,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,15,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,15,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,31,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,31,0.011610666910807291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,63,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,127,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,127,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,255,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,255,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,511,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,511,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,1023,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,1023,0.012778667112191519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,2047,0.017136000096797943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,2047,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,4095,0.01960533360640208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,4095,0.018992000569899876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,8191,0.02293866624434789
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,8191,0.021749332547187805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,16383,0.040448000033696495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,16383,0.027808000644048054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,1,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,1,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,float16,32767,0.06203199923038483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,1,128,0,1,float16,fp8,32767,0.043791999419530235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,3,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,3,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,7,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,7,0.011711999773979187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,15,0.011610666910807291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,15,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,31,0.011674666156371435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,63,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,31,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,63,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,127,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,127,0.011861333002646765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,255,0.0116799995303154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,511,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,511,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,1023,0.01349866638580958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,1023,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,2047,0.017045332739750545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,2047,0.01905599981546402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,4095,0.019178666174411774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,4095,0.021754667162895203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,8191,0.041877334316571556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,8191,0.025775998830795288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,16383,0.06081066528956095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,16383,0.04022933294375738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,1,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,float16,32767,0.10165866216023763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,1,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,3,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,3,0.011930666863918304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,2,128,0,1,float16,fp8,32767,0.06046933432420095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,7,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,7,0.01250133290886879
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,15,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,15,0.012346666306257248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,31,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,31,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,63,0.011621333658695221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,127,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,63,0.011637333780527115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,127,0.011999999483426413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,255,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,255,0.011957333733638128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,511,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,511,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,1023,0.013445333888133367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,2047,0.01932266727089882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,2047,0.020549333343903225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,1023,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,4095,0.0388373335202535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,4095,0.023546665906906128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,8191,0.05893866717815399
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,8191,0.03974399964014689
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,16383,0.10031466682751973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,16383,0.05871999760468801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,1,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,1,0.013594667116800943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,3,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,3,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,7,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,7,0.012698666503032049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,float16,32767,0.18142932653427124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,15,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,4,128,0,1,float16,fp8,32767,0.09768533706665039
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,15,0.012655999511480331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,31,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,31,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,63,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,63,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,127,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,127,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,255,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,255,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,511,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,511,0.01340266689658165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,1023,0.018709332992633183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,1023,0.016976000120242436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,2047,0.0397119993964831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,2047,0.023391999304294586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,4095,0.060975998640060425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,4095,0.03955200066169103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,8191,0.10140267014503479
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,8191,0.06062399844328562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,16383,0.1002400020758311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,16383,0.18401066462198892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,1,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,1,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,3,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,3,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,fp8,32767,0.18184000253677368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,16,8,128,0,1,float16,float16,32767,0.35046935081481934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,7,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,15,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,7,0.012223999947309494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,15,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,31,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,31,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,63,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,127,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,127,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,255,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,511,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,511,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,1023,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,1023,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,2047,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,2047,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,4095,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,4095,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,8191,0.01820266619324684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,8191,0.017466666797796886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,16383,0.021333334346612293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,16383,0.02146666745344798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,32767,0.02882666637500127
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,32767,0.03018666555484136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,65535,0.0356480007370313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,65535,0.032074667513370514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,1,0.010501333822806677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,1,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,3,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,3,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,7,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,15,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,float16,131071,0.038149334490299225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,15,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,63,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,63,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,127,0.01055466632048289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,1,128,0,1,float16,fp8,131071,0.03799466788768768
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,255,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,127,0.011600000162919363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,255,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,511,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,511,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,1023,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,1023,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,2047,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,2047,0.012533333152532578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,4095,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,4095,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,8191,0.016943999876578648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,8191,0.017498667041460674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,16383,0.01985599969824155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,16383,0.020928000410397846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,32767,0.0249493345618248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,32767,0.02325333406527837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,65535,0.02535466601451238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,1,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,1,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,3,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,65535,0.025205334027608235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,3,0.011952000359694162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,7,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,7,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,15,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,31,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,63,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,31,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,127,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,fp8,131071,0.027610667049884796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,255,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,255,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,2,128,0,1,float16,float16,131071,0.0316746657093366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,511,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,511,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,1023,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,1023,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,2047,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,2047,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,4095,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,4095,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,8191,0.01691199963291486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,8191,0.016986666868130367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,16383,0.019546666493018467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,16383,0.01844800015290578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,32767,0.021344001094500225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,32767,0.019317333896954853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,65535,0.025472000241279602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,65535,0.021194666624069214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,1,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,1,0.011770666887362799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,3,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,3,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,7,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,float16,131071,0.043562665581703186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,7,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,4,128,0,1,float16,fp8,131071,0.027610667049884796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,15,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,15,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,31,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,31,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,63,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,63,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,127,0.011578666667143503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,127,0.0116799995303154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,255,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,255,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,511,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,511,0.01166933278242747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,1023,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,1023,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,2047,0.0120319997270902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,4095,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,2047,0.013733333597580591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,4095,0.015967999895413715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,8191,0.017136000096797943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,8191,0.017173333714405697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,16383,0.01934933289885521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,16383,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,32767,0.023610666394233704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,32767,0.019530666371186573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,65535,0.03745600084463755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,65535,0.02345066765944163
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,1,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,1,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,3,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,float16,131071,0.05815466741720835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,7,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,7,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,15,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,16,8,128,0,1,float16,fp8,131071,0.039813332259655
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,15,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,31,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,31,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,63,0.01071999967098236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,63,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,127,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,127,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,255,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,511,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,255,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,511,0.01228800043463707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,1023,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,1023,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,2047,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,2047,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,4095,0.013530666629473368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,4095,0.013503999759753546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,8191,0.01918399954835574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,8191,0.020842666427294414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,16383,0.023541333774725597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,16383,0.021482666333516438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,32767,0.025466665625572205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,32767,0.025759999950726826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,65535,0.029557332396507263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,65535,0.02606933315594991
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,1,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,1,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,3,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,3,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,7,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,7,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,15,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,15,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,31,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,31,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,63,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,63,0.011744000017642975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,float16,131071,0.033439998825391136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,127,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,127,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,255,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,255,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,511,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,511,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,1,128,0,1,float16,fp8,131071,0.029509333272775013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,1023,0.012223999947309494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,1023,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,2047,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,2047,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,4095,0.0143306665122509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,4095,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,8191,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,8191,0.017616000026464462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,16383,0.01913600042462349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,16383,0.021151999632517498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,32767,0.023034666975339253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,32767,0.021013334393501282
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,65535,0.023247999449570973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,65535,0.025418666501839954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,1,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,1,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,3,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,3,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,7,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,float16,131071,0.042965332667032875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,7,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,15,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,31,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,15,0.011802667131026586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,31,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,63,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,63,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,127,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,127,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,2,128,0,1,float16,fp8,131071,0.029498666524887085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,255,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,255,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,511,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,511,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,1023,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,1023,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,2047,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,2047,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,4095,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,4095,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,8191,0.017680000513792038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,8191,0.017077332983414333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,16383,0.020058666666348774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,16383,0.019167999426523846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,32767,0.02332266668478648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,32767,0.019482667247454327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,65535,0.023498666783173878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,65535,0.0399893323580424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,1,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,1,0.011674666156371435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,3,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,3,0.011898666620254517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,7,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,float16,131071,0.060821334520975746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,7,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,15,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,15,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,31,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,63,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,4,128,0,1,float16,fp8,131071,0.041237334410349526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,127,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,255,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,511,0.011920000116030375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,1023,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,1023,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,2047,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,2047,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,4095,0.016634666671355564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,4095,0.016810666769742966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,8191,0.01826133330663045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,8191,0.017386666188637417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,16383,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,16383,0.02141333371400833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,32767,0.037962667644023895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,32767,0.023845332364241283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,65535,0.056794668237368263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,65535,0.037952000896135964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,1,0.011600000162919363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,3,0.01173866664369901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,7,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,7,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,15,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,31,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,fp8,131071,0.056890666484832764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,16,8,128,0,1,float16,float16,131071,0.09859733780225118
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,31,0.012709333250919977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,63,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,63,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,127,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,127,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,255,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,255,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,511,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,511,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,1023,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,1023,0.014159999787807465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,2047,0.01916266605257988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,2047,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,4095,0.02350933353106181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,4095,0.021327999730904896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,8191,0.04162666698296865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,8191,0.027621333797772724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,1,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,1,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,3,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,float16,16383,0.062047998110453285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,3,0.01340266689658165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,1,128,0,1,float16,fp8,16383,0.04413333535194397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,7,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,7,0.012080000092585882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,15,0.01249066616098086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,15,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,31,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,31,0.01231466606259346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,63,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,63,0.012778667112191519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,127,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,127,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,255,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,255,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,511,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,511,0.012768000364303589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,1023,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,1023,0.013455999394257864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,2047,0.021541332205136616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,2047,0.019280000279347103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,4095,0.04009066770474116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,4095,0.02388266722361247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,8191,0.06003733476003011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,8191,0.03881600002447764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,1,0.013637332866589228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,fp8,16383,0.058229332168896995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,2,128,0,1,float16,float16,16383,0.10085866848627727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,1,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,3,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,3,0.013541333377361298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,7,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,7,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,15,0.01349866638580958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,31,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,15,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,63,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,31,0.013679999858140945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,63,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,127,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,127,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,255,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,255,0.012719999998807907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,511,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,511,0.015360000232855478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,1023,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,1023,0.01766933376590411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,2047,0.04009599983692169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,2047,0.025034666061401367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,4095,0.06010666489601135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,4095,0.040250666439533234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,8191,0.10132267077763875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,8191,0.06026133398214976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,1,0.016650666793187458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,1,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,float16,16383,0.18266133467356363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,4,128,0,1,float16,fp8,16383,0.10122133294741313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,3,0.01643199970324834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,3,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,7,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,15,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,7,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,15,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,31,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,31,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,63,0.015493333339691162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,63,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,127,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,127,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,255,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,255,0.014922666052977243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,511,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,511,0.01926933353145917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,1023,0.03426666557788849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,1023,0.023168000082174938
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,2047,0.04063999901215235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,2047,0.06025599936644236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,4095,0.06150933106740316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,4095,0.10161067048708598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,8191,0.10232533017794292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,8191,0.1848906675974528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,1,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,1,0.013888000200192133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,3,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,3,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,7,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,fp8,16383,0.1834239959716797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,16,8,128,0,1,float16,float16,16383,0.35285866260528564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,31,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,31,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,63,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,63,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,127,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,127,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,255,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,255,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,511,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,511,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,1023,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,1023,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,2047,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,2047,0.014837333311637243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,4095,0.01534933348496755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,4095,0.014831999937693277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,8191,0.019695999721686046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,8191,0.019952000429232914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,16383,0.023370665808518726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,16383,0.021183999876181286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,32767,0.02390933285156886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,32767,0.023973333338896435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,65535,0.02733866622050603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,65535,0.025434667865435284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,1,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,1,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,3,0.011968000481526056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,3,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,float16,131071,0.04727466901143392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,7,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,7,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,1,128,0,1,float16,fp8,131071,0.032229334115982056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,15,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,15,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,31,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,31,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,63,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,63,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,127,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,127,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,255,0.011727999895811081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,511,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,511,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,1023,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,1023,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,2047,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,2047,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,4095,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,4095,0.017632000148296356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,8191,0.019274666905403137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,8191,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,16383,0.020960000654061634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,16383,0.02163200080394745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,32767,0.025493333737055462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,32767,0.021162666380405426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,65535,0.04064533362785975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,1,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,65535,0.025685332715511322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,1,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,3,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,3,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,7,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,15,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,15,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,31,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,31,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,63,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,63,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,float16,131071,0.062037333846092224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,127,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,255,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,511,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,2,128,0,1,float16,fp8,131071,0.04250133534272512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,511,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,1023,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,2047,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,2047,0.016890666137139004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,4095,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,4095,0.017386666188637417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,8191,0.019130667050679524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,8191,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,16383,0.02333866556485494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,16383,0.021269333859284718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,32767,0.04065066576004028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,32767,0.025941332181294758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,65535,0.03972266614437103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,65535,0.05938133100668589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,1,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,1,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,3,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,7,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,7,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,15,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,15,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,31,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,63,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,31,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,63,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,127,0.01173866664369901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,float16,131071,0.09961066643397014
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,4,128,0,1,float16,fp8,131071,0.060496002435684204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,255,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,511,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,511,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,1023,0.012560000022252401
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,1023,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,2047,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,2047,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,4095,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,4095,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,8191,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,8191,0.019130667050679524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,16383,0.03811199963092804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,16383,0.0232640008131663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,32767,0.05823466678460439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,32767,0.037962667644023895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,65535,0.09982400139172871
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,65535,0.05821333328882853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,1,0.012234666695197424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,1,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,3,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,float16,131071,0.1853653391202291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,3,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,7,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,7,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,16,8,128,0,1,float16,fp8,131071,0.09942932923634847
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,15,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,31,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,31,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,63,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,127,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,127,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,255,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,255,0.012608000387748083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,511,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,511,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,1023,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,1023,0.014885333677132925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,2047,0.021322667598724365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,2047,0.021941334009170532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,4095,0.04005866746107737
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,4095,0.025749333202838898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,1,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,1,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,3,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,3,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,float16,8191,0.06071466704209646
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,1,128,0,1,float16,fp8,8191,0.04273599882920583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,7,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,7,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,15,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,15,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,31,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,31,0.013381333400805792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,63,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,63,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,127,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,127,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,255,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,255,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,511,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,511,0.014736000448465347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,1023,0.01798933371901512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,1023,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,2047,0.04012266546487808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,2047,0.025285333395004272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,4095,0.0620000014702479
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,4095,0.04203199843565623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,1,0.015599999576807022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,1,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,3,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,float16,8191,0.10213333368301392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,2,128,0,1,float16,fp8,8191,0.0609493354956309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,3,0.015386667102575302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,7,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,7,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,15,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,15,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,31,0.015487999965747198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,31,0.016778666526079178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,63,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,63,0.014831999937693277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,127,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,127,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,255,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,255,0.014831999937693277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,511,0.019285333653291065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,511,0.01710933322707812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,1023,0.034927998979886375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,1023,0.02143466720978419
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,2047,0.06086933116118113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,2047,0.04091733445723852
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,4095,0.10168533523877461
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,4095,0.06227200229962667
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,1,0.019695999721686046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,1,0.017616000026464462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,float16,8191,0.1848319967587789
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,3,0.01972266659140587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,4,128,0,1,float16,fp8,8191,0.10316266616185506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,3,0.01801066721479098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,7,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,7,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,15,0.019434666881958645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,15,0.019472000499566395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,31,0.019002666076024372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,31,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,63,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,63,0.018895999838908512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,127,0.018986667195955913
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,127,0.017456000049908955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,255,0.01993600030740102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,255,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,511,0.033370666205883026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,511,0.023152001202106476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,1023,0.053344001372655235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,1023,0.03576533248027166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,2047,0.09568533301353455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,2047,0.058864002426465355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,4095,0.17150932550430298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,4095,0.09570133686065674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,fp8,8191,0.16690133015314737
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,16,8,128,0,1,float16,float16,8191,0.3249173363049825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,1,0.013429333766301474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,3,0.014773332824309668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,1,0.01350933313369751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,3,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,7,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,7,0.013552000125249227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,15,0.013642666240533194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,15,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,31,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,31,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,63,0.01431999976436297
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,63,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,127,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,127,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,255,0.013621332744757334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,255,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,511,0.01579733317097028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,511,0.016095999628305435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,1023,0.019445333629846573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,1023,0.018922666708628338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,1,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,float16,2047,0.042064001162846885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,1,128,0,1,float16,fp8,2047,0.02741866558790207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,1,0.016293333222468693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,3,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,3,0.015386667102575302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,7,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,7,0.01595199977358182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,15,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,15,0.01568000018596649
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,31,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,31,0.01568000018596649
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,63,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,63,0.015376000354687372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,127,0.015439999600251516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,127,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,255,0.01588800052801768
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,255,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,511,0.019760000209013622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,511,0.01714666684468587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,1023,0.02369600037733714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,1023,0.035829332967599235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,float16,2047,0.06258133550484975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,1,0.019626667102177937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,1,0.017781333376963932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,2,128,0,1,float16,fp8,2047,0.041663999358812966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,3,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,3,0.017765333255132038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,7,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,7,0.019391999890406925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,15,0.0195573332409064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,15,0.018757333358128864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,31,0.019626667102177937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,31,0.0194560003777345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,63,0.019498666127522785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,63,0.017871999492247898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,127,0.019386666516462963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,127,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,255,0.01939733326435089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,255,0.017290666699409485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,511,0.023386667172114056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,511,0.034602666894594826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,1023,0.05359466870625814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,1023,0.03430933256944021
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,1,0.02739199995994568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,float16,2047,0.09709333380063374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,1,0.025493333737055462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,4,128,0,1,float16,fp8,2047,0.05994133154551188
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,3,0.02740799884001414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,3,0.025461333493391674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,7,0.029285334050655365
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,7,0.025600001215934753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,15,0.0276853342851003
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,15,0.025402667621771496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,31,0.02922133356332779
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,31,0.025397333006064098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,63,0.027482666075229645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,63,0.025616000096003216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,127,0.027535999814669292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,127,0.02568000058333079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,255,0.033759998778502144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,255,0.025418666501839954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,511,0.053727999329566956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,511,0.03800000001986822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,1023,0.09106133381525676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,1023,0.054799998799959816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,fp8,2047,0.09630399942398071
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,16,8,128,0,1,float16,float16,2047,0.17271999518076578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,1,0.017498667041460674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,1,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,3,0.017749333133300144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,3,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,7,0.017263999829689663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,7,0.017344000438849132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,15,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,15,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,31,0.017551999539136887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,31,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,63,0.01758933315674464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,63,0.017397332936525345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,127,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,127,0.017664000391960144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,255,0.01692266638080279
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,255,0.01714666684468587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,511,0.021888000269730885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,511,0.019487999379634857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,float16,1023,0.03735466549793879
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,1,128,0,1,float16,fp8,1023,0.02587199956178665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,1,0.020197333147128422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,3,0.02117866774400075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,1,0.020501332978407543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,3,0.01923199991385142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,7,0.01926933353145917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,7,0.01978133370478948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,15,0.01913600042462349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,15,0.022250667214393616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,31,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,31,0.02000533292690913
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,63,0.01982933282852173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,127,0.019472000499566395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,63,0.019685332973798115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,127,0.018992000569899876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,255,0.019253333409627277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,255,0.01933866615096728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,511,0.03586666782697042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,511,0.023823998868465424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,float16,1023,0.0558186670144399
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,1,0.029530666768550873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,2,128,0,1,float16,fp8,1023,0.03531199942032496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,1,0.02595199892918269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,3,0.02942399928967158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,3,0.02605866640806198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,7,0.028736000259717304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,7,0.02565866708755493
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,15,0.0296426663796107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,15,0.025653332471847534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,31,0.028330666323502857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,31,0.02553066611289978
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,63,0.029359998802344005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,63,0.027215999861558277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,127,0.029882666965325672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,127,0.025727999707063038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,255,0.03397866586844126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,255,0.02737066646416982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,511,0.05413866539796194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,511,0.03783999880154928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,1,0.046522667010625206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,float16,1023,0.0934399962425232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,4,128,0,1,float16,fp8,1023,0.056133334835370384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,1,0.03994133323431015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,3,0.04625066618124644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,3,0.04106666644414266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,7,0.04611733555793762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,7,0.04146666576464971
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,15,0.04568000137805939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,15,0.0418453315893809
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,31,0.040021332601706185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,31,0.04621333380540212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,63,0.045935998360315956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,63,0.03968533376852671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,127,0.047824000318845115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,127,0.04041066765785217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,255,0.05292266607284546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,255,0.04308799902598063
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,511,0.06211199859778086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,511,0.09140800436337788
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,1,0.012768000364303589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,3,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,7,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,7,0.011994666109482447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,float16,1023,0.16695467631022134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,15,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,16,8,128,0,1,float16,fp8,1023,0.09523199995358785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,15,0.011744000017642975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,31,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,31,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,63,0.010469333579142889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,63,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,127,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,127,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,255,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,511,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,511,0.012741333494583765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,1023,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,1023,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,2047,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,2047,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,4095,0.019082666685183842
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,4095,0.018895999838908512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,8191,0.019551999866962433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,8191,0.01926933353145917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,16383,0.023056000471115112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,16383,0.02109333376089732
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,32767,0.02518933266401291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,32767,0.0233599990606308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,65535,0.041936000188191734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,65535,0.0296426663796107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,1,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,1,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,3,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,7,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,7,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,float16,131071,0.06377600133419037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,31,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,31,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,63,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,127,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,1,128,0,1,float16,fp8,131071,0.045909335215886436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,255,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,511,0.01268799975514412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,511,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,1023,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,1023,0.012256000190973282
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,2047,0.016735999534527462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,2047,0.017114666601022083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,4095,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,4095,0.017701332767804463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,8191,0.020351999749739964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,8191,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,16383,0.023845332364241283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,16383,0.02144533395767212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,32767,0.025258667767047882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,32767,0.04029333343108495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,65535,0.058975999553998314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,65535,0.04053866614898046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,1,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,1,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,3,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,3,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,7,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,7,0.011829332758982977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,15,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,31,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,15,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,31,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,63,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,63,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,127,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,127,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,fp8,131071,0.059893334905306496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,2,128,0,1,float16,float16,131071,0.10033599535624187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,511,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,511,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,1023,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,1023,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,2047,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,2047,0.015333333363135656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,4095,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,4095,0.017653333644072216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,8191,0.02146133283774058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,8191,0.0194560003777345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,16383,0.03989866624275843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,16383,0.023919999599456787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,32767,0.03866666555404663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,32767,0.05904000004132589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,65535,0.09968533118565877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,65535,0.059263999263445534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,1,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,3,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,1,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,3,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,7,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,15,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,15,0.012213333199421564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,31,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,31,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,63,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,63,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,127,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,fp8,131071,0.09956266482671101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,4,128,0,1,float16,float16,131071,0.18405866622924805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,255,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,255,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,511,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,511,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,1023,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,1023,0.01368533323208491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,2047,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,2047,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,4095,0.021456000705560047
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,4095,0.019088000059127808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,8191,0.03803733239571253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,8191,0.024160000185171764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,16383,0.060506666700045265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,16383,0.03783999880154928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,32767,0.05825066566467285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,32767,0.10070400436719258
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,65535,0.18323200941085815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,65535,0.09851200381914775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,1,0.02325333406527837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,float16,131071,0.34988800684611004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,1,0.023413332800070446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,3,0.023141334454218548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,16,8,128,0,1,float16,fp8,131071,0.1792373259862264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,3,0.02359466751416524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,7,0.023402666052182514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,7,0.02334933231274287
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,15,0.023386667172114056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,31,0.023455999791622162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,31,0.02367999901374181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,63,0.02383466561635335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,15,0.024512000381946564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,63,0.023397333920001984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,127,0.02348266790310542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,127,0.023573334018389385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,255,0.02402666707833608
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,255,0.023200000325838726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,float16,511,0.03763733307520548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,1,128,0,1,float16,fp8,511,0.029253333806991577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,1,0.03160533308982849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,1,0.029472000896930695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,3,0.031471999982992806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,3,0.028624000648657482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,7,0.031328000128269196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,7,0.029605334003766377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,15,0.03147733211517334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,15,0.028069332242012024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,31,0.031184000273545582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,31,0.029232000311215717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,63,0.02942399928967158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,63,0.03141333411137263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,127,0.030159999926884968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,127,0.029290666182835896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,255,0.03713600089152654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,255,0.02932800104220708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,float16,511,0.057445332407951355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,2,128,0,1,float16,fp8,511,0.04013866682847341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,1,0.04836800197760264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,1,0.04389866689840952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,3,0.048170665899912514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,3,0.04215466479460398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,7,0.048101335763931274
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,7,0.043605332573254905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,15,0.04830400149027506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,15,0.042037333051363625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,31,0.04799999793370565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,31,0.0436160018046697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,63,0.048207998275756836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,63,0.041834667325019836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,127,0.05013333261013031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,127,0.04167466859022776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,255,0.04588800172011057
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,255,0.05641599992911021
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,float16,511,0.09496532877286275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,4,128,0,1,float16,fp8,511,0.06250133117039998
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,1,0.08413333694140117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,1,0.070933332045873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,3,0.08370133241017659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,3,0.07071466743946075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,7,0.08418666323026021
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,7,0.07076266904671986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,15,0.08371733625729878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,15,0.0708426684141159
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,31,0.0849226713180542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,31,0.07128533224264781
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,63,0.0842080016930898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,63,0.070783997575442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,127,0.08401599526405334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,127,0.07417066891988118
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,255,0.09434666236241658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,255,0.07444799939791362
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,fp8,511,0.10983999570210774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,16,8,128,0,1,float16,float16,511,0.1699413259824117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,1,0.037802666425704956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,1,0.03581333408753077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,3,0.037621334195137024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,3,0.03612799942493439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,7,0.0359253336985906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,7,0.035631999373435974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,15,0.0377813329299291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,15,0.035599999129772186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,31,0.03818666686614355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,31,0.036015999813874565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,63,0.037962667644023895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,63,0.03616533428430557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,127,0.038015998899936676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,127,0.03570666660865148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,fp8,255,0.03682666768630346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,1,128,0,1,float16,float16,255,0.042117332418759666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,1,0.05243733525276184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,1,0.046426668763160706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,3,0.05226133267084757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,3,0.04649066428343455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,7,0.052485331892967224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,7,0.04587199787298838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,15,0.05231466889381409
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,15,0.04604266583919525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,31,0.05207466582457224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,31,0.04611733555793762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,63,0.05231466889381409
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,63,0.04595200220743815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,127,0.05462400118509928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,127,0.047242666284243263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,float16,255,0.06078400214513143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,1,0.08719999591509502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,1,0.07645866771539052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,2,128,0,1,float16,fp8,255,0.05004799862702688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,3,0.08728532989819844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,3,0.07700799902280171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,7,0.08721599976221721
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,7,0.07659199833869934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,15,0.08729066451390584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,15,0.0764213353395462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,31,0.0869653324286143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,31,0.07643199960390727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,63,0.0881173312664032
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,63,0.07675200204054515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,127,0.08878933389981587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,127,0.07893333335717519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,float16,255,0.10069333513577779
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,1,0.1551359991232554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,1,0.1359946628411611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,4,128,0,1,float16,fp8,255,0.08141866823037465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,3,0.15666666626930237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,3,0.1362826625506083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,7,0.15652267138163248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,7,0.13622933626174927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,15,0.15627732872962952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,15,0.13648533821105957
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,31,0.156031996011734
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,31,0.136272003253301
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,63,0.15619200468063354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,63,0.1362773378690084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,127,0.1364479959011078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,127,0.15566933155059814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,1,0.012453333785136541
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,1,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,3,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,3,0.01166933278242747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,7,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,7,0.01239466667175293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,15,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,15,0.012650666137536367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,31,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,31,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,63,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,float16,255,0.17697600523630777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,63,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,127,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,127,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,255,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,255,0.011834666132926941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,511,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,511,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,16,8,128,0,1,float16,fp8,255,0.1402666668097178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,1023,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,1023,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,2047,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,2047,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,4095,0.019088000059127808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,4095,0.018986667195955913
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,8191,0.02163200080394745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,8191,0.019359999646743137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,16383,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,16383,0.023200000325838726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,32767,0.04222933451334635
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,32767,0.027658666173617046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,1,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,3,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,3,0.011727999895811081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,7,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,15,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,7,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,15,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,float16,65535,0.061247999469439186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,1,128,0,1,float16,fp8,65535,0.043791999419530235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,31,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,63,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,127,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,127,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,255,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,255,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,511,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,511,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,1023,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,1023,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,2047,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,2047,0.01552533358335495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,4095,0.017594666530688603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,4095,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,8191,0.021295999487241108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,8191,0.019850666324297588
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,16383,0.04011200120051702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,16383,0.02518933266401291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,32767,0.059392000238100685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,32767,0.0395413339138031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,1,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,1,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,3,0.012042666474978128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,float16,65535,0.10095999638239543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,3,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,7,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,7,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,15,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,2,128,0,1,float16,fp8,65535,0.05759466687838236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,31,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,31,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,63,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,127,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,127,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,255,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,255,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,511,0.0124746672809124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,511,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,1023,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,1023,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,2047,0.01718933383623759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,2047,0.016864000509182613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,4095,0.021125334004561108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,4095,0.019296000401178997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,8191,0.03921599934498469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,8191,0.023647998770078022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,16383,0.059674665331840515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,16383,0.03958400090535482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,32767,0.0592853327592214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,32767,0.10122666756312053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,1,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,1,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,3,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,3,0.011776000261306763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,7,0.011802667131026586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,7,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,15,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,float16,65535,0.18212266763051352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,15,0.011850666254758835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,4,128,0,1,float16,fp8,65535,0.09840533137321472
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,31,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,31,0.011989332735538483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,63,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,63,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,127,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,127,0.012250666817029318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,255,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,255,0.011765333513418833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,511,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,511,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,1023,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,1023,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,2047,0.018789333601792652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,2047,0.020207999895016353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,4095,0.03765333443880081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,4095,0.023792001108328503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,8191,0.03732266773780187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,8191,0.05874133110046387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,16383,0.058133333921432495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,16383,0.09967999656995137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,32767,0.09725333253542583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,32767,0.18154666821161905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,1,0.010698666175206503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,1,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,1,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,3,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,3,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,3,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,3,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,7,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,fp8,65535,0.17542932430903116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,7,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,16,8,128,0,1,float16,float16,65535,0.34702932834625244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,7,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,7,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,15,0.010565333068370819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,15,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,31,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,31,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,31,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,31,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,63,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,63,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,63,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,63,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,127,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,127,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,255,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,255,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,255,0.011658667276302973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,255,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,511,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,511,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,511,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,511,0.012624000509579977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,1023,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,1023,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,1023,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,2047,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,1023,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,2047,0.015498666713635126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,2047,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,2047,0.015423999478419622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,4095,0.013541333377361298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,4095,0.016837333639462788
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,4095,0.01350933313369751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,4095,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,8191,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,8191,0.019786667078733444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,8191,0.01392000044385592
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,8191,0.01915733392039935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,16383,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,16383,0.025466665625572205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,16383,0.014373333503802618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,16383,0.02235200007756551
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,32767,0.040218666195869446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,32767,0.013557333499193192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,32767,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,32767,0.03170666595300039
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,float16,65535,0.01360000049074491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,1,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,float16,65535,0.06007466713587443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,128,1,float16,fp8,65535,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,1,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,1,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,3,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,3,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,3,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,3,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,64,0,1,float16,fp8,65535,0.05231466889381409
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,7,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,7,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,15,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,15,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,15,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,15,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,31,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,31,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,31,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,31,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,63,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,63,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,63,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,63,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,127,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,127,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,255,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,255,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,255,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,511,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,511,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,511,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,511,0.012778667112191519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,1023,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,1023,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,1023,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,1023,0.013445333888133367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,2047,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,2047,0.016965333372354507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,2047,0.01479999969402949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,2047,0.016864000509182613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,4095,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,4095,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,4095,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,4095,0.01937599976857503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,8191,0.01451733335852623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,8191,0.025242666403452556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,8191,0.015114666273196539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,8191,0.023775999744733173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,16383,0.01470400020480156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,16383,0.03969600051641464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,16383,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,16383,0.031744000812371574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,32767,0.013349333157142004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,32767,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,32767,0.06098133325576782
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,32767,0.0524586687485377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,float16,65535,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,float16,65535,0.10212266445159912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,1,0.01173866664369901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,1,0.011690666278203329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,128,1,float16,fp8,65535,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,1,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,1,0.012234666695197424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,3,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,64,0,1,float16,fp8,65535,0.08710400263468425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,3,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,7,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,7,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,7,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,15,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,15,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,15,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,15,0.012351999680201212
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,31,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,31,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,31,0.011663999408483505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,63,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,31,0.012495999534924826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,63,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,63,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,63,0.01221866657336553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,127,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,127,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,127,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,127,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,255,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,255,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,255,0.012229333321253458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,255,0.011578666667143503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,511,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,511,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,511,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,511,0.011674666156371435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,1023,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,1023,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,1023,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,1023,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,2047,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,2047,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,2047,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,2047,0.017632000148296356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,4095,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,4095,0.02364266663789749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,4095,0.014954666296641031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,4095,0.021317332983016968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,8191,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,8191,0.037808001041412354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,8191,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,8191,0.031045332551002502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,16383,0.05831466615200043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,16383,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,16383,0.014783999572197596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,16383,0.051498666405677795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,32767,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,32767,0.0997226635615031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,32767,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,32767,0.08662399649620056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,float16,65535,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,1,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,float16,65535,0.1822986602783203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,128,1,float16,fp8,65535,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,1,0.010656000425418219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,64,0,1,float16,fp8,65535,0.15549332896868387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,1,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,3,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,3,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,3,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,7,0.010527999450763067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,7,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,7,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,15,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,15,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,15,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,15,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,31,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,31,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,31,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,63,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,63,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,127,0.010581333190202713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,63,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,127,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,127,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,255,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,255,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,255,0.011690666278203329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,511,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,511,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,511,0.011616000284751257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,511,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,1023,0.010656000425418219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,1023,0.011781333635250727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,1023,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,1023,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,2047,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,2047,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,2047,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,2047,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,4095,0.012719999998807907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,4095,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,4095,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,4095,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,8191,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,8191,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,8191,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,8191,0.015658666690190632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,16383,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,16383,0.019205333044131596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,16383,0.01956266661485036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,16383,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,32767,0.02739733209212621
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,32767,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,32767,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,32767,0.02734400083621343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,65535,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,65535,0.03176533430814743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,65535,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,65535,0.029653333127498627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,float16,131071,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,float16,131071,0.03344533344109853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,1,0.010533332824707031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,1,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,1,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,128,1,float16,fp8,131071,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,3,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,3,0.01246400053302447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,3,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,7,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,7,0.012618667135636011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,7,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,15,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,15,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,15,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,15,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,31,0.010575999816258749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,31,0.012736000120639801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,31,0.010591999938090643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,31,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,64,0,1,float16,fp8,131071,0.03397866586844126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,63,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,63,0.010608000059922537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,63,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,127,0.011781333635250727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,127,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,127,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,127,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,255,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,255,0.011760000139474869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,255,0.013722666849692663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,255,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,511,0.010442666709423065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,511,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,511,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,511,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,1023,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,1023,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,1023,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,1023,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,2047,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,2047,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,2047,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,2047,0.01267733300725619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,4095,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,4095,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,4095,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,4095,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,8191,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,8191,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,8191,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,8191,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,16383,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,16383,0.02145066608985265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,16383,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,16383,0.0195573332409064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,32767,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,32767,0.022970666488011677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,32767,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,32767,0.024085332949956257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,65535,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,65535,0.025472000241279602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,65535,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,65535,0.02317333221435547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,float16,131071,0.01351999988158544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,float16,131071,0.027317332724730175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,1,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,128,1,float16,fp8,131071,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,1,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,3,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,1,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,3,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,3,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,3,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,7,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,7,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,15,0.010575999816258749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,7,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,15,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,15,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,15,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,64,0,1,float16,fp8,131071,0.02569599946339925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,31,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,31,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,31,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,31,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,63,0.010602666685978571
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,63,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,63,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,127,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,127,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,127,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,127,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,255,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,255,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,255,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,511,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,511,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,511,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,1023,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,511,0.013679999858140945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,1023,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,1023,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,1023,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,2047,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,2047,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,2047,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,2047,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,4095,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,4095,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,4095,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,4095,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,8191,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,8191,0.017488000293572743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,8191,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,8191,0.015770666301250458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,16383,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,16383,0.019130667050679524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,16383,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,16383,0.017637333522240322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,32767,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,32767,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,32767,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,32767,0.019589333484570186
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,65535,0.021498667697111767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,65535,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,65535,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,65535,0.02103466788927714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,float16,131071,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,1,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,float16,131071,0.028165332973003387
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,128,1,float16,fp8,131071,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,1,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,64,0,1,float16,fp8,131071,0.02716800073782603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,1,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,1,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,3,0.010512000570694605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,3,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,3,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,3,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,7,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,7,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,15,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,15,0.010581333190202713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,15,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,15,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,31,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,31,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,31,0.010586666564146677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,63,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,63,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,63,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,127,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,127,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,127,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,255,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,255,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,255,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,511,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,511,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,511,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,511,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,1023,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,1023,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,1023,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,1023,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,2047,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,2047,0.011493333925803503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,2047,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,2047,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,4095,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,4095,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,4095,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,4095,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,8191,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,8191,0.017093333105246227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,8191,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,8191,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,16383,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,16383,0.020266667008399963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,16383,0.013557333499193192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,16383,0.02088533341884613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,32767,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,32767,0.023189333577950794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,32767,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,32767,0.023631999890009563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,65535,0.024405332903067272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,65535,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,65535,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,65535,0.023552000522613525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,float16,131071,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,float16,131071,0.02792000025510788
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,128,1,float16,fp8,131071,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,1,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,1,0.01184533288081487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,1,0.011493333925803503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,3,0.012149333953857422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,3,0.010453333457310995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,3,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,7,0.010527999450763067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,3,0.012624000509579977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,7,0.010538666198650995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,15,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,7,0.012437333663304647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,15,0.010725333044926325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,31,0.010549332946538925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,15,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,31,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,31,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,64,0,1,float16,fp8,131071,0.02733866622050603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,31,0.01191466674208641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,63,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,127,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,127,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,127,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,127,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,255,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,255,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,255,0.01232533281048139
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,255,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,511,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,511,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,511,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,511,0.011823999385039011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,1023,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,1023,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,1023,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,1023,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,2047,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,2047,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,2047,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,2047,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,4095,0.01339200014869372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,4095,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,4095,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,4095,0.01350933313369751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,8191,0.01201066623131434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,8191,0.01709866647919019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,8191,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,8191,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,16383,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,16383,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,16383,0.019882666567961376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,16383,0.019130667050679524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,32767,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,32767,0.021562665700912476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,32767,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,32767,0.019354666272799175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,65535,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,65535,0.02195200075705846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,65535,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,65535,0.021242665747801464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,float16,131071,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,float16,131071,0.028549333413441975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,1,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,1,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,3,0.010602666685978571
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,128,1,float16,fp8,131071,0.013541333377361298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,3,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,3,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,7,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,7,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,15,0.010591999938090643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,15,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,15,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,15,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,64,0,1,float16,fp8,131071,0.02731200059254964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,31,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,31,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,31,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,63,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,63,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,63,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,127,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,127,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,127,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,255,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,255,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,255,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,511,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,511,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,511,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,511,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,1023,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,1023,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,1023,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,1023,0.012655999511480331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,2047,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,2047,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,2047,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,2047,0.012709333250919977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,4095,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,4095,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,4095,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,4095,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,8191,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,8191,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,8191,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,8191,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,16383,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,16383,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,16383,0.01870399961868922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,16383,0.017530667285124462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,32767,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,32767,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,32767,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,32767,0.019498666127522785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,65535,0.023370665808518726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,65535,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,65535,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,65535,0.023386667172114056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,float16,131071,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,128,1,float16,fp8,131071,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,float16,131071,0.040762667854626976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,64,0,1,float16,fp8,131071,0.03170666595300039
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,1,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,1,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,3,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,3,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,7,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,7,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,7,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,7,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,15,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,15,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,31,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,31,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,31,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,31,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,63,0.010687999427318573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,63,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,63,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,63,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,127,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,127,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,127,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,127,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,255,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,255,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,255,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,255,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,511,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,511,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,511,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,511,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,1023,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,1023,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,1023,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,1023,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,2047,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,2047,0.01728533332546552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,2047,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,2047,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,4095,0.013573333621025085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,4095,0.019194666296243668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,4095,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,4095,0.019482667247454327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,8191,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,8191,0.025413334369659424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,8191,0.013658666362365087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,8191,0.02329600105683009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,16383,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,16383,0.013616000612576803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,16383,0.038719999293486275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,16383,0.03173333406448364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,float16,32767,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,float16,32767,0.06073066592216492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,1,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,1,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,1,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,128,1,float16,fp8,32767,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,3,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,1,0.012645332763592402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,3,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,3,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,3,0.01239466667175293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,7,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,7,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,7,0.011855999628702799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,7,0.014053333550691605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,15,0.01163200040658315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,64,0,1,float16,fp8,32767,0.05223466455936432
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,15,0.011616000284751257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,15,0.01228800043463707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,15,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,31,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,31,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,31,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,63,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,63,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,63,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,63,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,127,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,127,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,127,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,255,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,255,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,255,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,511,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,511,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,511,0.012736000120639801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,511,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,1023,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,1023,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,1023,0.01209066684047381
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,1023,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,2047,0.014064000298579534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,2047,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,2047,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,2047,0.01905599981546402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,4095,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,4095,0.023567999402681988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,4095,0.014736000448465347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,4095,0.023061332603295643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,8191,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,8191,0.01452800010641416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,8191,0.039749334255854286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,8191,0.0316746657093366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,16383,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,16383,0.060746664802233376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,16383,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,16383,0.05260799825191498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,float16,32767,0.01381333296497663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,float16,32767,0.1013813316822052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,1,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,1,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,1,0.012682666381200155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,1,0.012639999389648438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,128,1,float16,fp8,32767,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,3,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,3,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,3,0.013503999759753546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,3,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,7,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,64,0,1,float16,fp8,32767,0.08716799815495808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,7,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,7,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,7,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,15,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,15,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,15,0.011834666132926941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,15,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,31,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,31,0.011674666156371435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,31,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,31,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,63,0.012437333663304647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,63,0.012693333129088083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,63,0.013455999394257864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,63,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,127,0.01211200033624967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,127,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,127,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,127,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,255,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,255,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,255,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,511,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,511,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,255,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,511,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,511,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,1023,0.012714666624863943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,1023,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,1023,0.017658667018016178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,1023,0.017759999881188076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,2047,0.015509333461523056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,2047,0.023589332898457844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,2047,0.023503998915354412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,2047,0.01605333387851715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,4095,0.015450666348139444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,4095,0.03946666667858759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,4095,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,4095,0.03279466678698858
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,8191,0.060975998640060425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,8191,0.015477333217859268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,8191,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,8191,0.053930665055910744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,16383,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,16383,0.10442666212717693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,16383,0.01540800059835116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,16383,0.08748799562454224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,float16,32767,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,1,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,float16,32767,0.18779732783635458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,128,1,float16,fp8,32767,0.015477333217859268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,1,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,1,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,1,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,3,0.010384000216921171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,3,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,64,0,1,float16,fp8,32767,0.15492266416549683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,3,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,3,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,7,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,7,0.010586666564146677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,7,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,7,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,15,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,15,0.010464000205198923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,15,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,15,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,31,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,31,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,31,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,63,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,127,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,127,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,127,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,127,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,255,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,255,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,255,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,511,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,511,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,511,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,511,0.012357333054145178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,1023,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,1023,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,1023,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,1023,0.012714666624863943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,2047,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,2047,0.013365333278973898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,2047,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,2047,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,4095,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,4095,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,4095,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,4095,0.015386667102575302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,8191,0.01349866638580958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,8191,0.017621333400408428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,8191,0.013658666362365087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,8191,0.018543999642133713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,16383,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,16383,0.01945066700379054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,16383,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,16383,0.021125334004561108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,32767,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,32767,0.02327999969323476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,32767,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,32767,0.021338666478792827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,65535,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,65535,0.023333333432674408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,65535,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,65535,0.023445333043734234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,float16,131071,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,1,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,float16,131071,0.02939733366171519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,1,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,1,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,1,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,3,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,128,1,float16,fp8,131071,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,3,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,7,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,7,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,15,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,15,0.010538666198650995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,31,0.010538666198650995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,31,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,31,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,64,0,1,float16,fp8,131071,0.02923733244339625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,63,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,63,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,63,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,63,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,127,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,127,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,127,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,127,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,255,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,255,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,255,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,511,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,511,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,511,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,511,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,1023,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,1023,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,1023,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,1023,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,2047,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,2047,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,2047,0.013642666240533194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,2047,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,4095,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,4095,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,4095,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,4095,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,8191,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,8191,0.019296000401178997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,8191,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,8191,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,16383,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,16383,0.019466667125622433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,16383,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,16383,0.019306667149066925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,32767,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,32767,0.02139200021823247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,32767,0.021375998854637146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,32767,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,65535,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,65535,0.02609066665172577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,65535,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,65535,0.0236160010099411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,float16,131071,0.042064001162846885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,float16,131071,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,1,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,1,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,1,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,1,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,128,1,float16,fp8,131071,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,3,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,3,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,3,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,7,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,7,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,7,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,7,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,15,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,15,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,64,0,1,float16,fp8,131071,0.03379199902216593
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,15,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,15,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,31,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,31,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,31,0.01166933278242747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,63,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,31,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,63,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,63,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,63,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,127,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,127,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,127,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,127,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,255,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,255,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,255,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,511,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,511,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,511,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,511,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,1023,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,1023,0.012080000092585882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,1023,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,1023,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,2047,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,2047,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,2047,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,2047,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,4095,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,4095,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,4095,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,4095,0.017136000096797943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,8191,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,8191,0.018320000420014065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,8191,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,8191,0.017450666675964992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,16383,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,16383,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,16383,0.02125866711139679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,16383,0.02073066681623459
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,32767,0.013482666263977686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,32767,0.025397333006064098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,32767,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,32767,0.024330665667851765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,65535,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,65535,0.03953066716591517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,65535,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,65535,0.031717332700888314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,float16,131071,0.0592853327592214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,float16,131071,0.01586666703224182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,128,1,float16,fp8,131071,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,1,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,64,0,1,float16,fp8,131071,0.05274133384227753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,1,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,1,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,1,0.012666666259368261
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,3,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,3,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,3,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,3,0.012522666404644648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,7,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,7,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,7,0.01166933278242747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,15,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,15,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,15,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,15,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,31,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,31,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,31,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,31,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,63,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,63,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,63,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,127,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,127,0.012202666451533636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,127,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,127,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,255,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,255,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,255,0.01211200033624967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,511,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,511,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,511,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,511,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,1023,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,1023,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,1023,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,1023,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,2047,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,2047,0.019530666371186573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,2047,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,2047,0.018805333723624546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,4095,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,4095,0.023397333920001984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,4095,0.01534933348496755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,4095,0.02170666555563609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,8191,0.014592000593741735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,8191,0.03772266705830892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,8191,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,8191,0.03156800071398417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,float16,16383,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,float16,16383,0.06030400097370148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,1,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,128,1,float16,fp8,16383,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,1,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,1,0.013637332866589228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,1,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,3,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,3,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,3,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,3,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,64,0,1,float16,fp8,16383,0.05198933184146881
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,7,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,7,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,7,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,7,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,15,0.012768000364303589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,15,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,15,0.01369599997997284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,15,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,31,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,31,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,31,0.01351999988158544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,63,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,63,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,63,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,127,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,127,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,63,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,127,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,127,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,255,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,255,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,255,0.012725333372751871
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,255,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,511,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,511,0.014458666245142618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,511,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,511,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,1023,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,1023,0.017338667064905167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,1023,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,1023,0.016885332763195038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,2047,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,2047,0.01532799998919169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,2047,0.023130667706330616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,2047,0.023530667026837666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,4095,0.01544533297419548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,4095,0.039861333866914116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,4095,0.015402667224407196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,4095,0.03182933231194814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,8191,0.015498666713635126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,8191,0.060922667384147644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,8191,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,8191,0.0524533341328303
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,float16,16383,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,128,1,float16,fp8,16383,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,float16,16383,0.10340799887975057
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,64,0,1,float16,fp8,16383,0.08742933471997578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,1,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,1,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,1,0.013658666362365087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,1,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,3,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,3,0.01349866638580958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,3,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,3,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,7,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,7,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,7,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,7,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,15,0.01339200014869372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,15,0.013781332721312841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,15,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,31,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,15,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,31,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,31,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,31,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,63,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,63,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,63,0.01404800017674764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,63,0.015114666273196539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,127,0.013621332744757334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,127,0.013541333377361298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,127,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,127,0.013754667093356451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,255,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,255,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,255,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,255,0.013557333499193192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,511,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,511,0.01781333362062772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,511,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,511,0.01703466723362605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,1023,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,1023,0.023141334454218548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,1023,0.014831999937693277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,1023,0.02179199953873952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,2047,0.03972266614437103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,2047,0.017877332866191864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,2047,0.01773333301146825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,2047,0.03155199935038885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,4095,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,4095,0.017397332936525345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,4095,0.0621973325808843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,4095,0.05410666763782501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,8191,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,8191,0.10402133067448933
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,8191,0.01720000058412552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,8191,0.08834667007128398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,float16,16383,0.017701332767804463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,float16,16383,0.18780267238616943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,128,1,float16,fp8,16383,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,1,0.013663999736309052
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,1,0.011600000162919363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,1,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,1,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,3,0.013653332988421122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,3,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,3,0.01198400060335795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,64,0,1,float16,fp8,16383,0.156058669090271
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,3,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,7,0.012655999511480331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,7,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,7,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,15,0.01240533341964086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,7,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,15,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,15,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,15,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,31,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,31,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,31,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,31,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,63,0.012778667112191519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,63,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,63,0.013653332988421122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,63,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,127,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,127,0.012719999998807907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,127,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,127,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,255,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,255,0.011855999628702799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,255,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,255,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,511,0.01179733375708262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,511,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,511,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,511,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,1023,0.01202133297920227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,1023,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,1023,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,1023,0.015344000111023584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,2047,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,2047,0.025040000677108765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,2047,0.015439999600251516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,2047,0.02306666721900304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,float16,4095,0.015376000354687372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,float16,4095,0.04016000032424927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,128,1,float16,fp8,4095,0.01540800059835116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,64,0,1,float16,fp8,4095,0.031541332602500916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,1,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,1,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,1,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,1,0.014736000448465347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,3,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,3,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,3,0.014773332824309668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,3,0.013786666095256805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,7,0.01479999969402949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,7,0.014869333555301031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,7,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,7,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,15,0.01488000030318896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,15,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,15,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,15,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,31,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,31,0.014954666296641031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,31,0.015333333363135656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,31,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,63,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,63,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,63,0.01481066644191742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,63,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,127,0.01552533358335495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,127,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,127,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,127,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,255,0.01403733342885971
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,255,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,255,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,255,0.01488000030318896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,511,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,511,0.017450666675964992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,511,0.014831999937693277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,511,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,1023,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,1023,0.022357332209746044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,1023,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,1023,0.021205333371957142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,2047,0.017450666675964992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,2047,0.03988266736268997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,2047,0.017162666966517765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,2047,0.033520000676314034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,float16,4095,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,128,1,float16,fp8,4095,0.017674667139848072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,float16,4095,0.062128002444903054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,1,0.01720533271630605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,1,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,1,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,3,0.017407999684413273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,64,0,1,float16,fp8,4095,0.053743998209635414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,3,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,1,0.01830400029818217
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,3,0.017184000462293625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,3,0.018917333334684372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,7,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,7,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,7,0.017551999539136887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,7,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,15,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,15,0.017466666797796886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,15,0.017237332959969837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,15,0.018506667266289394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,31,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,31,0.017653333644072216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,31,0.017562666287024815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,63,0.01743999992807706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,31,0.018101333330074947
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,63,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,63,0.017690667261679966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,63,0.017263999829689663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,127,0.017674667139848072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,127,0.017338667064905167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,127,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,127,0.017456000049908955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,255,0.017973333597183228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,255,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,255,0.017290666699409485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,255,0.017594666530688603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,511,0.02126399924357732
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,511,0.017952000101407368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,511,0.01757866640885671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,511,0.021226666867733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,1023,0.01749333366751671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,1023,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,1023,0.03545066714286804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,1023,0.029616000751654308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,2047,0.021146667500336964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,2047,0.05716800192991892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,2047,0.021226666867733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,2047,0.05133866767088572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,float16,4095,0.021349333226680756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,float16,4095,0.09530666470527649
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,0,1,float16,fp8,4095,0.08112533390522003
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,64,128,1,float16,fp8,4095,0.021301334102948506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,1,0.014741333822409311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,1,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,1,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,1,0.014741333822409311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,3,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,3,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,3,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,3,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,7,0.014778666198253632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,7,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,7,0.014890667051076889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,7,0.014954666296641031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,15,0.015402667224407196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,15,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,15,0.014752000570297241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,15,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,31,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,31,0.015370666980743408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,31,0.014885333677132925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,31,0.015573333948850632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,63,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,63,0.014671999961137772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,63,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,63,0.015754666179418564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,127,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,127,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,127,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,127,0.014906667172908783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,255,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,255,0.014709333578745524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,255,0.014933332800865173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,255,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,511,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,511,0.017077332983414333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,511,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,511,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,1023,0.014890667051076889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,1023,0.02165333429972331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,1023,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,1023,0.021146667500336964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,float16,2047,0.017237332959969837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,float16,2047,0.0402453343073527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,128,1,float16,fp8,2047,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,1,0.017642666896184284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,1,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,1,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,1,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,3,0.017136000096797943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,64,0,1,float16,fp8,2047,0.03367999941110611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,3,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,3,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,3,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,7,0.017573333034912746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,7,0.017621333400408428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,7,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,7,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,15,0.017610666652520496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,15,0.017045332739750545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,15,0.01732800031701724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,15,0.017317333569129307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,31,0.017722666263580322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,31,0.017557332913080852
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,31,0.017237332959969837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,31,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,63,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,63,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,63,0.01720533271630605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,63,0.01752000053723653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,127,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,127,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,127,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,127,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,255,0.018778666853904724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,255,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,255,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,255,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,511,0.0189280000825723
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,511,0.023237332701683044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,511,0.018042666216691334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,511,0.02186666677395503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,1023,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,1023,0.033733333150545754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,1023,0.018122666825850803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,1023,0.02945599953333537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,float16,2047,0.021189334491888683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,float16,2047,0.05798399945100149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,128,1,float16,fp8,2047,0.01958400011062622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,64,0,1,float16,fp8,2047,0.050517335534095764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,1,0.025418666501839954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,1,0.025445332129796345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,1,0.023381332556406658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,1,0.023813332120577495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,3,0.025098666548728943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,3,0.0252960001428922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,3,0.023226665953795116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,3,0.02367466688156128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,7,0.02526933451493581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,7,0.025216000775496166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,7,0.02346666653951009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,7,0.025306666890780132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,15,0.025072000920772552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,15,0.02346133440732956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,15,0.02514133354028066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,15,0.025077333052953083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,31,0.025173333783944447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,31,0.02348266790310542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,31,0.02532266577084859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,31,0.02516799916823705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,63,0.02513599892457326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,63,0.023413332800070446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,63,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,63,0.026677332818508148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,127,0.025477332373460133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,127,0.023658665517965954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,127,0.025311999022960663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,127,0.025253333151340485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,255,0.025466665625572205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,255,0.025093334416548412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,255,0.025199999411900837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,255,0.025546667476495106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,511,0.025546667476495106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,511,0.03793066740036011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,511,0.025631998976071674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,511,0.033386667569478355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,1023,0.02619733413060506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,1023,0.056613331039746605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,1023,0.026000000536441803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,1023,0.050255998969078064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,float16,2047,0.027642667293548584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,float16,2047,0.09744532903035481
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,128,1,float16,fp8,2047,0.02718399961789449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,1,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,1,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,1,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,1,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,3,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,3,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,7,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,3,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,7,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,7,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,64,0,1,float16,fp8,2047,0.08416000008583069
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,15,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,15,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,15,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,15,0.011610666910807291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,31,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,31,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,31,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,31,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,63,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,63,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,63,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,63,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,127,0.010677333921194077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,127,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,127,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,255,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,127,0.012367999802033106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,255,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,255,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,255,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,511,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,511,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,511,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,511,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,1023,0.010543999572594961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,1023,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,1023,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,1023,0.01232533281048139
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,2047,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,2047,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,2047,0.013712000101804733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,2047,0.013861333330472311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,4095,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,4095,0.015247999380032221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,4095,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,4095,0.017194667210181553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,8191,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,8191,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,8191,0.017375999440749485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,8191,0.01748266691962878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,16383,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,16383,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,16383,0.019039999693632126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,16383,0.019253333409627277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,32767,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,32767,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,32767,0.02109866589307785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,32767,0.021146667500336964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,65535,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,65535,0.025829332570234936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,65535,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,65535,0.025733334322770435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,float16,131071,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,1,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,1,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,float16,131071,0.04181866844495138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,128,1,float16,fp8,131071,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,64,0,1,float16,fp8,131071,0.03347733368476232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,3,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,3,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,3,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,7,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,3,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,7,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,7,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,7,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,15,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,15,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,15,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,31,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,31,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,31,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,31,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,63,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,63,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,63,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,63,0.01180800050497055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,127,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,127,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,127,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,127,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,255,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,255,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,255,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,255,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,511,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,511,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,511,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,511,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,1023,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,1023,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,1023,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,2047,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,1023,0.0116799995303154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,2047,0.016293333222468693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,2047,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,2047,0.015333333363135656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,4095,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,4095,0.01414399966597557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,4095,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,4095,0.016917333006858826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,8191,0.013786666095256805
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,8191,0.018976000448067982
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,8191,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,8191,0.01759999990463257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,16383,0.021061333517233532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,16383,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,16383,0.013418667018413544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,16383,0.019226666539907455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,32767,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,32767,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,32767,0.025685332715511322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,32767,0.025946666797002155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,65535,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,65535,0.0400693342089653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,65535,0.01340266689658165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,65535,0.031557333966096245
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,float16,131071,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,float16,131071,0.05901333192984263
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,1,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,1,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,1,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,1,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,128,1,float16,fp8,131071,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,3,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,3,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,3,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,3,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,7,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,7,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,7,0.011658667276302973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,7,0.01180800050497055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,15,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,15,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,15,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,64,0,1,float16,fp8,131071,0.05327466626962026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,31,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,31,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,31,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,63,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,63,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,63,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,63,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,127,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,127,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,255,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,255,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,255,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,255,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,511,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,511,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,511,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,511,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,1023,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,1023,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,1023,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,1023,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,2047,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,2047,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,2047,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,2047,0.01470400020480156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,4095,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,4095,0.01553600033124288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,4095,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,4095,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,8191,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,8191,0.01887999971707662
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,8191,0.01339200014869372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,8191,0.01762666677435239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,16383,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,16383,0.02384000023206075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,16383,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,16383,0.02139200021823247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,32767,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,32767,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,32767,0.038047999143600464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,32767,0.031888000667095184
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,65535,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,65535,0.059018666545550026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,65535,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,65535,0.05218133330345154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,float16,131071,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,float16,131071,0.10176533460617065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,128,1,float16,fp8,131071,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,1,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,1,0.01884799947341283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,3,0.019120000302791595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,64,0,1,float16,fp8,131071,0.0890933374563853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,1,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,1,0.0179626668492953
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,3,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,3,0.017290666699409485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,3,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,7,0.018005333840847015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,7,0.01889066646496455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,7,0.01720000058412552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,7,0.018895999838908512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,15,0.017466666797796886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,15,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,15,0.017717332889636356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,15,0.01887999971707662
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,31,0.019626667102177937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,31,0.017375999440749485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,31,0.017237332959969837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,31,0.019061333189407986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,63,0.018933333456516266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,63,0.017488000293572743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,63,0.01905599981546402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,63,0.016949333250522614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,127,0.018250666558742523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,127,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,127,0.01759999990463257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,127,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,255,0.0191040001809597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,255,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,255,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,255,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,511,0.019093333433071773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,511,0.02327999969323476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,511,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,511,0.023706667125225067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,float16,1023,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,float16,1023,0.03498133271932602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,128,1,float16,fp8,1023,0.017488000293572743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,64,0,1,float16,fp8,1023,0.029994666576385498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,1,0.025392000873883564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,1,0.02571733295917511
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,1,0.0252960001428922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,1,0.025445332129796345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,3,0.025258667767047882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,3,0.025631998976071674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,3,0.025349333882331848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,3,0.02510933329661687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,7,0.025248001019159954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,7,0.025248001019159954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,7,0.025429333249727886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,7,0.025397333006064098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,15,0.025439999997615814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,15,0.025045332809289295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,15,0.025663999219735462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,15,0.02515733242034912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,31,0.025333332518736523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,31,0.025413334369659424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,31,0.02535466601451238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,31,0.02532800038655599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,63,0.02516266703605652
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,63,0.02568000058333079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,63,0.025301332275072735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,63,0.02533866713444392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,127,0.02513599892457326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,127,0.02531733363866806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,127,0.025514667232831318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,127,0.025392000873883564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,255,0.025834667185942333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,255,0.02587199956178665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,255,0.025536000728607178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,255,0.025536000728607178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,511,0.03733866661787033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,511,0.025839999318122864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,511,0.02569066733121872
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,511,0.0336053321758906
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,float16,1023,0.025349333882331848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,float16,1023,0.055861334005991616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,128,1,float16,fp8,1023,0.02569599946339925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,1,0.03957866628964742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,1,0.04021333406368891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,64,0,1,float16,fp8,1023,0.05011733373006185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,1,0.037471999724706016
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,1,0.03811199963092804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,3,0.03976533313592275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,3,0.039674667020638786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,3,0.0382080003619194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,3,0.037690666814645134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,7,0.03989866624275843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,7,0.039749334255854286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,7,0.03775999943415324
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,7,0.03803733239571253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,15,0.039701332648595176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,15,0.04166933397452036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,15,0.03772799919048945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,15,0.03813866774241129
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,31,0.039621333281199135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,31,0.03899200012286504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,31,0.037962667644023895
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,31,0.038165333370367684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,63,0.039461334546407066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,63,0.03775466730197271
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,63,0.03824000060558319
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,63,0.03847466657559077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,127,0.03941333293914795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,127,0.03939199944337209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,127,0.03791466603676478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,255,0.04221333563327789
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,127,0.03800000001986822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,255,0.04165866722663244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,255,0.03979733337958654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,255,0.03793599953254064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,511,0.042352000872294106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,511,0.060090666015942894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,511,0.03896533449490865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,511,0.056741332014401756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,float16,1023,0.04197333256403605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,float16,1023,0.0950933297475179
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,128,1,float16,fp8,1023,0.03817066550254822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,64,0,1,float16,fp8,1023,0.08530666430791219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,1,0.02624533325433731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,1,0.025754667818546295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,1,0.025663999219735462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,1,0.025653332471847534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,3,0.02624533325433731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,3,0.025600001215934753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,3,0.025946666797002155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,3,0.02607999990383784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,7,0.025792000194390614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,7,0.025424001117547352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,7,0.025813333690166473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,7,0.025733334322770435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,15,0.02622933437426885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,15,0.025248001019159954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,15,0.025514667232831318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,15,0.026309333741664886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,31,0.026127999027570088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,31,0.025701334079106648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,31,0.02605333427588145
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,31,0.025920001169045765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,63,0.026608000199000042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,63,0.025450666745503742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,63,0.025813333690166473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,63,0.026000000536441803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,127,0.025802666942278545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,127,0.025744001070658367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,127,0.025605333348115284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,255,0.027914665639400482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,255,0.02603733291228612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,255,0.025754667818546295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,255,0.025573333104451496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,float16,511,0.027744000156720478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,511,0.037765334049860634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,128,1,float16,fp8,511,0.025802666942278545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,float16,127,0.026144000391165417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,1,0.04033066580692927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,64,0,1,float16,fp8,511,0.03509333233038584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,1,0.0403413325548172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,1,0.03928533444801966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,3,0.03995733211437861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,1,0.03982933362325033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,3,0.040362666050593056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,3,0.039135999977588654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,3,0.039477333426475525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,7,0.03979733337958654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,7,0.040234667559464775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,7,0.03988266736268997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,7,0.03882133215665817
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,15,0.039450667798519135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,15,0.040063999593257904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,31,0.03946666667858759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,31,0.04011200120051702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,31,0.03867733230193456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,31,0.04004266609748205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,63,0.03991466760635376
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,63,0.03965866565704346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,63,0.040522667268911995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,63,0.037903999288876854
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,15,0.04052799940109253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,15,0.040037333965301514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,127,0.03977599988381068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,127,0.04009066770474116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,127,0.04010133445262909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,127,0.0377813329299291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,255,0.042165334026018776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,255,0.043231998880704246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,255,0.039647998909155525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,255,0.039887999494870506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,float16,511,0.0415040006240209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,float16,511,0.06065066655476888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,128,1,float16,fp8,511,0.03963200002908707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,1,0.06814399858315785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,1,0.06828799843788147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,64,0,1,float16,fp8,511,0.05795733133951823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,1,0.06463466584682465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,3,0.06866133213043213
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,1,0.06438399851322174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,3,0.06869866450627644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,3,0.06643733382225037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,7,0.06879466772079468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,7,0.0682773341735204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,7,0.06683200101057689
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,7,0.06654400130112965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,15,0.0684853345155716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,15,0.06913599868615468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,15,0.06691733499368031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,15,0.06633600095907848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,31,0.06614399949709575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,31,0.0690773328145345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,31,0.06891733407974243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,31,0.06454933186372121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,3,0.06578133503595988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,63,0.06891199946403503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,63,0.06900266806284587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,63,0.06474133332570393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,127,0.06993600229422252
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,127,0.07088000078996022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,63,0.0672159989674886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,127,0.06469866633415222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,127,0.06528533498446147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,255,0.07508799930413564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,255,0.07113066812356313
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,255,0.0666720022757848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,255,0.06782400111357371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,float16,511,0.07502933343251546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,float16,511,0.10847999652226765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,1,0.0116799995303154
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,1,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,1,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,1,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,3,0.011637333780527115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,3,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,3,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,7,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,128,1,float16,fp8,511,0.0665226678053538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,7,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,7,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,15,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,15,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,64,0,1,float16,fp8,511,0.09928533434867859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,15,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,15,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,31,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,31,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,31,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,63,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,63,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,127,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,127,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,127,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,127,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,255,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,255,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,255,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,511,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,511,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,511,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,511,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,1023,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,1023,0.012170666207869848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,1023,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,1023,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,2047,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,2047,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,2047,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,2047,0.015114666273196539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,4095,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,4095,0.01695999999841054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,4095,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,4095,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,8191,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,8191,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,8191,0.018922666708628338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,8191,0.01749333366751671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,16383,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,16383,0.02130666623512904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,16383,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,16383,0.021125334004561108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,32767,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,32767,0.025407999753952026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,32767,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,32767,0.023904000719388325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,65535,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,65535,0.04018666595220566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,65535,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,65535,0.0313226655125618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,float16,131071,0.01482133318980535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,1,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,float16,131071,0.06061333417892456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,1,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,1,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,1,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,3,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,3,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,3,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,3,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,7,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,7,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,128,1,float16,fp8,131071,0.014842666685581207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,7,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,7,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,64,0,1,float16,fp8,131071,0.05194133520126343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,15,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,15,0.010559999694426855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,15,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,31,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,15,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,31,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,31,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,63,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,31,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,63,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,127,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,63,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,63,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,127,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,255,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,255,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,255,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,255,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,511,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,511,0.012245333443085352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,511,0.012661332885424295
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,511,0.012736000120639801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,1023,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,1023,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,1023,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,1023,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,2047,0.01340266689658165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,2047,0.014906667172908783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,2047,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,2047,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,4095,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,4095,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,4095,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,4095,0.017018667111794155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,8191,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,8191,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,8191,0.018965333700180054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,8191,0.019386666516462963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,16383,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,16383,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,16383,0.02500266581773758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,16383,0.02314666658639908
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,32767,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,32767,0.037530665596326195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,32767,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,32767,0.031914666295051575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,65535,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,65535,0.05871999760468801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,65535,0.013637332866589228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,65535,0.05217066903909048
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,float16,131071,0.014752000570297241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,1,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,128,1,float16,fp8,131071,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,float16,131071,0.1018933355808258
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,1,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,64,0,1,float16,fp8,131071,0.08896000186602275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,1,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,1,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,3,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,3,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,3,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,7,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,7,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,7,0.011674666156371435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,15,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,15,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,15,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,15,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,31,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,31,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,31,0.01179733375708262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,63,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,63,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,63,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,63,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,127,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,127,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,127,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,127,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,255,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,255,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,255,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,511,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,511,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,511,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,511,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,1023,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,1023,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,1023,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,1023,0.013605333864688873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,2047,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,2047,0.013418667018413544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,2047,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,2047,0.015344000111023584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,4095,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,4095,0.019178666174411774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,4095,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,4095,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,8191,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,8191,0.023621333142121632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,8191,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,8191,0.023311999936898548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,16383,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,16383,0.013455999394257864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,16383,0.039887999494870506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,16383,0.03187733391920725
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,32767,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,32767,0.06029866635799408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,32767,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,32767,0.05208533505598704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,65535,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,65535,0.10008533795674641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,65535,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,65535,0.08695466319719951
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,float16,131071,0.1846933364868164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,float16,131071,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,1,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,1,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,128,1,float16,fp8,131071,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,7,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,3,0.012015999605258306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,7,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,15,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,15,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,31,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,64,0,1,float16,fp8,131071,0.1575146714846293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,31,0.011760000139474869
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,63,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,63,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,127,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,127,0.011802667131026586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,511,0.011792000383138657
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,511,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,1023,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,1023,0.011621333658695221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,2047,0.018538666268189747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,2047,0.01703466723362605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,4095,0.01904533306757609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,4095,0.016949333250522614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,8191,0.019519999623298645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,8191,0.02149333308140437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,16383,0.04036800066630045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,16383,0.025018667181332905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,32767,0.06140799820423126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,32767,0.039733332892258964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,1,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,3,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,7,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,15,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,15,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,31,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,31,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,float16,65535,0.10288000106811523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,63,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,63,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,1,128,0,1,float16,fp8,65535,0.058703998724619545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,127,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,255,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,127,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,511,0.011823999385039011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,511,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,1023,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,1023,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,2047,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,2047,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,4095,0.021568000316619873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,4095,0.019306667149066925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,8191,0.03965866565704346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,8191,0.025498665869235992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,16383,0.06108266611893972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,16383,0.040031999349594116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,32767,0.10148266951243083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,32767,0.058575997749964394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,1,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,1,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,float16,65535,0.18195732434590658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,3,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,3,0.011850666254758835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,7,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,15,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,7,0.011776000261306763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,15,0.012213333199421564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,31,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,31,0.012693333129088083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,2,128,0,1,float16,fp8,65535,0.09796266754468282
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,63,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,63,0.012416000167528788
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,127,0.012154666086037954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,127,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,255,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,255,0.012309333930412928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,511,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,511,0.013749333719412485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,1023,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,1023,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,2047,0.021338666478792827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,2047,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,4095,0.03852266569932302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,4095,0.025706666211287182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,8191,0.059706668059031166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,8191,0.03789333254098892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,16383,0.10069866975148518
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,16383,0.05815466741720835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,32767,0.1811786691347758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,32767,0.09724799791971843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,1,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,3,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,1,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,3,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,7,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,7,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,15,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,15,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,31,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,fp8,65535,0.17558934291203818
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,31,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,63,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,8,4,128,0,1,float16,float16,65535,0.34322667121887207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,127,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,255,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,255,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,511,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,511,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,1023,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,1023,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,2047,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,2047,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,4095,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,4095,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,8191,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,8191,0.015509333461523056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,16383,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,16383,0.020261333634455998
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,32767,0.027402666707833607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,32767,0.028079998989899952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,65535,0.03289066751797994
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,65535,0.0322773332397143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,1,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,1,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,3,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,3,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,7,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,7,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,15,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,15,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,31,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,31,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,63,0.010725333044926325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,float16,131071,0.03595733394225439
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,63,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,127,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,127,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,255,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,1,128,0,1,float16,fp8,131071,0.034661332766215004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,255,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,511,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,511,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,1023,0.01227733368674914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,1023,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,2047,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,2047,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,4095,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,4095,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,8191,0.016832000265518825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,8191,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,16383,0.02027733375628789
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,16383,0.021130666136741638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,32767,0.023445333043734234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,32767,0.021925332645575207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,65535,0.027621333797772724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,65535,0.02532266577084859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,1,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,1,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,3,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,3,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,float16,131071,0.031445334355036415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,7,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,15,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,31,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,31,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,63,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,2,128,0,1,float16,fp8,131071,0.028442665934562683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,127,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,127,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,511,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,511,0.01192533348997434
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,1023,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,1023,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,2047,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,2047,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,4095,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,4095,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,8191,0.01701333373785019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,8191,0.016858667135238647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,16383,0.019487999379634857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,16383,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,32767,0.021482666333516438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,32767,0.021295999487241108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,65535,0.02548266698916753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,1,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,65535,0.02207999924818675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,1,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,3,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,3,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,7,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,7,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,15,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,31,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,15,0.011930666863918304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,31,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,63,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,float16,131071,0.043738668163617454
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,8,4,128,0,1,float16,fp8,131071,0.02743999908367793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,63,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,127,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,127,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,255,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,511,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,511,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,1023,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,1023,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,2047,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,2047,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,4095,0.013557333499193192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,4095,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,8191,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,8191,0.017573333034912746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,16383,0.02160000056028366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,16383,0.021087999145189922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,32767,0.0252960001428922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,32767,0.02332799881696701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,65535,0.025439999997615814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,65535,0.02734400083621343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,1,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,1,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,3,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,3,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,7,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,7,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,15,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,15,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,31,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,31,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,63,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,63,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,127,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,127,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,fp8,131071,0.02828266719977061
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,1,128,0,1,float16,float16,131071,0.031386665999889374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,255,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,255,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,511,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,511,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,1023,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,1023,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,2047,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,2047,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,4095,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,4095,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,8191,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,8191,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,16383,0.019194666296243668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,16383,0.019434666881958645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,32767,0.02309866746266683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,32767,0.021189334491888683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,65535,0.025631998976071674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,65535,0.023418667415777843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,1,0.011653333902359009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,1,0.011674666156371435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,3,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,float16,131071,0.04557866851488749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,3,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,7,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,7,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,15,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,15,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,31,0.010469333579142889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,63,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,31,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,2,128,0,1,float16,fp8,131071,0.02956266701221466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,127,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,127,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,255,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,255,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,511,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,1023,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,1023,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,2047,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,2047,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,4095,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,4095,0.015626666446526844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,8191,0.01728533332546552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,8191,0.01701333373785019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,16383,0.02187199890613556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,16383,0.01932266727089882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,32767,0.02332266668478648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,32767,0.021349333226680756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,65535,0.04015466570854187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,65535,0.02531733363866806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,1,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,3,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,3,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,7,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,7,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,15,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,31,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,15,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,31,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,63,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,63,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,float16,131071,0.06058666606744131
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,127,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,255,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,511,0.012442667037248611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,8,4,128,0,1,float16,fp8,131071,0.04085333396991094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,255,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,511,0.01219733307758967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,1023,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,1023,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,2047,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,2047,0.017050666113694508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,4095,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,4095,0.01932266727089882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,8191,0.04075733323891958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,8191,0.02587733417749405
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,16383,0.05995733539263407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,16383,0.04044266790151596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,1,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,3,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,1,0.011637333780527115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,3,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,7,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,7,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,15,0.01173866664369901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,float16,32767,0.1018293301264445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,15,0.011973333855470022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,1,128,0,1,float16,fp8,32767,0.05941333373387655
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,31,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,31,0.012117333710193634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,63,0.01179733375708262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,63,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,127,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,127,0.012223999947309494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,255,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,255,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,511,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,511,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,1023,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,1023,0.013855999956528345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,2047,0.021583999196688335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,2047,0.0194560003777345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,4095,0.03962666789690653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,4095,0.023578666150569916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,8191,0.03905066599448522
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,8191,0.06162666777769724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,16383,0.10129599769910176
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,16383,0.05795200169086456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,1,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,1,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,3,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,float16,32767,0.18281066417694092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,3,0.013408000270525614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,7,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,7,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,2,128,0,1,float16,fp8,32767,0.09730133414268494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,15,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,15,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,31,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,31,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,63,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,63,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,127,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,127,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,255,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,255,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,511,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,511,0.014831999937693277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,1023,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,1023,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,2047,0.03982933362325033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,2047,0.02366400013367335
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,4095,0.03990400085846583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,4095,0.06112533311049143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,8191,0.10086400310198466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,8191,0.06039466460545858
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,16383,0.18492799997329712
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,1,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,16383,0.10075199604034424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,1,0.011834666132926941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,3,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,3,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,7,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,7,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,15,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,15,0.011711999773979187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,float16,32767,0.34697067737579346
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,31,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,63,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,63,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,127,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,127,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,255,0.010549332946538925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,255,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,8,4,128,0,1,float16,fp8,32767,0.18254399299621582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,511,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,511,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,1023,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,1023,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,2047,0.01482133318980535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,2047,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,4095,0.01584533353646596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,4095,0.015360000232855478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,8191,0.019152000546455383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,8191,0.018906666586796444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,16383,0.02146666745344798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,16383,0.021253332495689392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,32767,0.022986667851607006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,32767,0.0249439999461174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,65535,0.023408000667889912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,65535,0.02638400097688039
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,1,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,1,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,3,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,7,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,7,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,15,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,15,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,31,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,31,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,63,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,63,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,127,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,127,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,float16,131071,0.04704533517360687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,255,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,1,128,0,1,float16,fp8,131071,0.02959466725587845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,255,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,511,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,511,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,1023,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,1023,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,2047,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,2047,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,4095,0.01681600014368693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,4095,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,8191,0.019546666493018467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,8191,0.01877333347996076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,16383,0.021338666478792827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,16383,0.020997333029905956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,32767,0.02462399999300639
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,32767,0.021514666577180225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,65535,0.04162666698296865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,65535,0.025920001169045765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,1,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,1,0.011839999506870905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,3,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,3,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,7,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,float16,131071,0.06246933341026306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,7,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,15,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,15,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,2,128,0,1,float16,fp8,131071,0.04387733340263367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,31,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,31,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,63,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,63,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,127,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,255,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,511,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,511,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,1023,0.011989332735538483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,1023,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,2047,0.014682666709025701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,2047,0.016895999511082966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,4095,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,4095,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,8191,0.018800000349680584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,8191,0.020581333587567013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,16383,0.02372266600529353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,16383,0.021146667500336964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,32767,0.04140799989302953
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,32767,0.02553066611289978
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,65535,0.059530665477116905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,65535,0.039733332892258964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,1,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,1,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,3,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,3,0.011829332758982977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,7,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,7,0.012506666282812754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,float16,131071,0.0993280013402303
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,15,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,15,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,31,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,8,4,128,0,1,float16,fp8,131071,0.06028266747792562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,63,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,63,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,127,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,255,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,127,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,255,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,511,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,511,0.012698666503032049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,1023,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,1023,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,2047,0.021498667697111767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,2047,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,4095,0.041493333876132965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,4095,0.025386666258176167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,8191,0.060864001512527466
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,8191,0.03979733337958654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,1,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,1,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,3,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,float16,16383,0.10126399993896484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,3,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,7,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,1,128,0,1,float16,fp8,16383,0.0587360014518102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,7,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,15,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,15,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,31,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,31,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,63,0.012741333494583765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,63,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,127,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,127,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,255,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,255,0.01258133351802826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,511,0.014650666465361914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,511,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,1023,0.017349333812793095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,1023,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,2047,0.039546666045983635
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,2047,0.025018667181332905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,4095,0.04145599901676178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,4095,0.06080000102519989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,8191,0.06053866446018219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,8191,0.1018506685892741
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,1,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,1,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,float16,16383,0.18424532810846964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,3,0.01569066693385442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,3,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,7,0.015589332828919092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,7,0.01568000018596649
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,2,128,0,1,float16,fp8,16383,0.10260799527168274
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,15,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,15,0.01481066644191742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,31,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,31,0.015413332730531693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,63,0.015471999843915304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,63,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,127,0.015626666446526844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,127,0.015482666591803232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,255,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,255,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,511,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,511,0.019530666371186573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,1023,0.02147199958562851
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,1023,0.03561066587766012
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,2047,0.061424002051353455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,2047,0.04021333406368891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,4095,0.1020960013071696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,4095,0.06293333570162456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,8191,0.1843573252360026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,8191,0.10412266850471497
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,1,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,float16,16383,0.3503093322118123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,8,4,128,0,1,float16,fp8,16383,0.18532800674438477
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,1,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,3,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,3,0.012709333250919977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,7,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,15,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,15,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,7,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,31,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,31,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,63,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,63,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,127,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,127,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,255,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,255,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,511,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,511,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,1023,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,1023,0.016858667135238647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,2047,0.04020266731580099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,2047,0.025813333690166473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,1,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,float16,4095,0.06049066781997681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,1,128,0,1,float16,fp8,4095,0.04090133309364319
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,3,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,1,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,3,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,7,0.01563199982047081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,7,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,15,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,15,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,31,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,31,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,63,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,63,0.014831999937693277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,127,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,127,0.01479999969402949
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,255,0.01533866673707962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,255,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,511,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,511,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,1023,0.03532266616821289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,1023,0.021194666624069214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,2047,0.0620000014702479
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,2047,0.042021334171295166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,1,0.019717333217461903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,1,0.018842666099468868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,float16,4095,0.10339732964833577
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,2,128,0,1,float16,fp8,4095,0.06243733565012614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,3,0.019509332875410717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,3,0.019120000302791595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,7,0.01915733392039935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,7,0.018063999712467194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,15,0.019199999670187633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,15,0.019215999792019527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,31,0.018965333700180054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,31,0.017605333278576534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,63,0.018944000204404194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,63,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,127,0.019391999890406925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,127,0.017093333105246227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,255,0.01939733326435089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,255,0.01728533332546552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,511,0.03365866591533025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,511,0.023077333966890972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,1023,0.055498664577802025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,1023,0.03399466723203659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,2047,0.0962506632010142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,2047,0.05937600135803223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,1,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,float16,4095,0.17092265685399374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,8,4,128,0,1,float16,fp8,4095,0.0960693359375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,1,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,3,0.016757333030303318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,3,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,7,0.015610666324694952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,7,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,15,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,15,0.015520000209410986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,31,0.015450666348139444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,31,0.015840000162522
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,63,0.015658666690190632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,63,0.015381333728631338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,127,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,127,0.015552000453074774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,255,0.01533866673707962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,255,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,511,0.019343999524911244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,511,0.0170666662355264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,1023,0.03566399961709976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,1023,0.023333333432674408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,fp8,2047,0.04110399881998698
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,1,0.019445333629846573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,1,128,0,1,float16,float16,2047,0.06208533545335134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,1,0.018496000518401463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,3,0.019519999623298645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,3,0.017498667041460674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,7,0.01820266619324684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,7,0.020010666300853092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,15,0.019551999866962433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,15,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,31,0.019317333896954853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,63,0.01932799940307935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,31,0.019727999965349834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,127,0.01989866668979327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,63,0.01748266691962878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,127,0.017738666385412216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,255,0.020586666961510975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,255,0.017194667210181553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,511,0.03334933271010717
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,511,0.023573334018389385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,1023,0.03418133407831192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,1023,0.05527466535568237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,float16,2047,0.09748799602190654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,1,0.027562665442625683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,2,128,0,1,float16,fp8,2047,0.058730666836102806
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,1,0.025386666258176167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,3,0.029440000653266907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,3,0.025461333493391674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,7,0.027466667195161183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,7,0.025498665869235992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,15,0.029109333952267964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,15,0.025914666553338368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,31,0.027503999571005504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,31,0.025568000972270966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,63,0.029194665451844532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,63,0.02537599951028824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,127,0.027914665639400482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,127,0.025557334224383037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,255,0.03359466542800268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,255,0.025818665822347004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,511,0.05232533315817515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,511,0.03773866593837738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,1023,0.09165333708127339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,1023,0.05479466418425242
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,1,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,3,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,fp8,2047,0.09591999650001526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,7,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,8,4,128,0,1,float16,float16,2047,0.1725813349088033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,15,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,31,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,31,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,63,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,63,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,127,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,127,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,255,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,255,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,511,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,511,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,1023,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,1023,0.01268799975514412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,2047,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,2047,0.014762666076421738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,4095,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,4095,0.017114666601022083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,8191,0.019285333653291065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,8191,0.019386666516462963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,16383,0.02149333308140437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,16383,0.02060266708334287
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,32767,0.02479466547568639
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,32767,0.021770666042963665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,65535,0.041264000038305916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,65535,0.02681066592534383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,1,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,1,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,3,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,3,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,7,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,float16,131071,0.06275733311971028
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,15,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,31,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,31,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,63,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,63,0.011690666278203329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,127,0.010506667196750641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,255,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,127,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,1,128,0,1,float16,fp8,131071,0.04305600126584371
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,511,0.01173866664369901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,511,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,1023,0.011999999483426413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,1023,0.011754666765530905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,2047,0.016623999923467636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,2047,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,4095,0.019018666197856266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,4095,0.01727466657757759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,8191,0.01985599969824155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,8191,0.019493332753578823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,16383,0.02515200028816859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,16383,0.021941334009170532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,32767,0.02534399926662445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,32767,0.04170133173465729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,65535,0.05968533456325531
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,65535,0.04119999955097834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,1,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,3,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,3,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,7,0.010656000425418219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,7,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,15,0.010565333068370819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,float16,131071,0.10169600447018941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,15,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,31,0.010496000448862711
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,31,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,63,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,63,0.011706666400035223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,127,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,127,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,255,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,255,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,511,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,1023,0.012645332763592402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,511,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,2047,0.015487999965747198
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,1023,0.01259200026591619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,2047,0.01681600014368693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,2,128,0,1,float16,fp8,131071,0.0603359987338384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,4095,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,4095,0.01710933322707812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,8191,0.02142400046189626
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,8191,0.019215999792019527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,16383,0.03879466652870178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,16383,0.02348800003528595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,32767,0.05842666824658712
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,32767,0.03794133414824804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,65535,0.09940800070762634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,65535,0.05880533158779144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,1,0.019920000185569126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,1,0.019546666493018467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,3,0.019424000134070713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,fp8,131071,0.09931199749310811
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,8,4,128,0,1,float16,float16,131071,0.18360000848770142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,3,0.01953599974513054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,7,0.019391999890406925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,7,0.019226666539907455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,15,0.019386666516462963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,15,0.01953599974513054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,31,0.02110933264096578
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,31,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,63,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,63,0.01932799940307935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,127,0.020997333029905956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,127,0.019237333287795384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,255,0.021269333859284718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,255,0.019274666905403137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,511,0.03692266593376795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,511,0.024031999210516613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,1,0.029658667743206024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,float16,1023,0.054416000843048096
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,1,128,0,1,float16,fp8,1023,0.035349334279696144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,1,0.02735999971628189
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,3,0.029338667790095013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,3,0.02752000093460083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,7,0.029792000850041706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,7,0.027327999472618103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,15,0.029509333272775013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,15,0.025472000241279602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,31,0.02977066735426585
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,31,0.02734400083621343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,63,0.02979733298222224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,63,0.026357332865397137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,127,0.030048000315825146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,127,0.027402666707833607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,255,0.03420799970626831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,255,0.027024000883102417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,511,0.05469333132108053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,511,0.03835200021664301
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,float16,1023,0.09345066547393799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,2,128,0,1,float16,fp8,1023,0.056159997979799904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,1,0.046282668908437095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,1,0.039893334110577904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,3,0.046154667933781944
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,3,0.0420959989229838
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,7,0.046021332343419395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,7,0.03961600114901861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,15,0.04624533156553904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,15,0.03973866750796636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,31,0.03996799886226654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,63,0.045663997530937195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,63,0.03997333347797394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,31,0.04646400113900503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,127,0.03976000100374222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,127,0.04788800080617269
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,255,0.05417066812515259
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,255,0.04283200204372406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,511,0.060592000683148704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,511,0.09106133381525676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,float16,1023,0.16648000478744507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,8,4,128,0,1,float16,fp8,1023,0.09532800316810608
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,1,0.029391999046007793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,1,0.028463999430338543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,3,0.03140799949566523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,3,0.028794666131337483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,7,0.02942399928967158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,7,0.028202667832374573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,15,0.0315733328461647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,15,0.02788266787926356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,31,0.03159466634194056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,31,0.027632000545660656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,63,0.031445334355036415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,63,0.0278613343834877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,127,0.03142400085926056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,127,0.027535999814669292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,255,0.03722133239110311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,255,0.029215998947620392
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,1,0.04839999973773956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,float16,511,0.05762133498986562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,1,128,0,1,float16,fp8,511,0.040031999349594116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,1,0.04201599955558777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,3,0.048101335763931274
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,3,0.04234666625658671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,7,0.04831466575463613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,7,0.042250668009122215
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,15,0.04790399968624115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,15,0.0423573354880015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,31,0.04799999793370565
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,31,0.04234133164087931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,63,0.04789866507053375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,63,0.042277331153551735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,127,0.050554667909940086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,127,0.043968002001444496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,255,0.056218668818473816
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,255,0.04643199841181437
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,float16,511,0.09542933106422424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,1,0.08301866551240285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,2,128,0,1,float16,fp8,511,0.06338666876157124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,1,0.07120533287525177
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,3,0.08531199892361958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,3,0.0724533349275589
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,7,0.08504533767700195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,7,0.07257066667079926
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,15,0.08528000116348267
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,15,0.07249600191911061
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,31,0.08538666367530823
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,31,0.07316266496976216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,63,0.08518399794896443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,63,0.07262399792671204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,127,0.08496000369389851
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,127,0.07489066819349925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,255,0.09550399581591289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,255,0.07708799839019775
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,1,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,1,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,3,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,3,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,7,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,7,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,15,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,15,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,fp8,511,0.10983999570210774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,8,4,128,0,1,float16,float16,511,0.16893333196640015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,31,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,31,0.011701333026091257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,63,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,127,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,63,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,127,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,255,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,511,0.012186666329701742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,511,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,1023,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,1023,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,2047,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,2047,0.017103999853134155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,4095,0.018735999862353008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,4095,0.017338667064905167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,8191,0.019695999721686046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,8191,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,16383,0.023541333774725597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,16383,0.02160000056028366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,32767,0.025279998779296875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,32767,0.04261866708596548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,65535,0.05991999804973602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,65535,0.04116799930731455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,1,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,1,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,3,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,3,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,7,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,7,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,15,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,15,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,31,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,float16,131071,0.10211732983589172
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,31,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,63,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,127,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,63,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,127,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,255,0.011578666667143503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,511,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,511,0.011733333269755045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,1,128,0,1,float16,fp8,131071,0.060506666700045265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,1023,0.012634667257467905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,1023,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,2047,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,2047,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,4095,0.01918399954835574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,4095,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,8191,0.02146133283774058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,8191,0.019989332805077236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,16383,0.04012266546487808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,16383,0.02332799881696701
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,32767,0.038746667404969536
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,32767,0.06061866879463196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,65535,0.058543999989827476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,65535,0.10010133186976115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,1,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,1,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,3,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,3,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,7,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,7,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,15,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,31,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,31,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,63,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,63,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,127,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,127,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,255,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,255,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,float16,131071,0.18473066886266074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,511,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,511,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,2,128,0,1,float16,fp8,131071,0.10086933771769206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,1023,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,1023,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,2047,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,2047,0.017077332983414333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,4095,0.021407999098300934
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,4095,0.0195573332409064
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,8191,0.0401706670721372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,8191,0.025248001019159954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,16383,0.06003733476003011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,16383,0.03946666667858759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,32767,0.09955199559529622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,32767,0.05856533348560333
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,65535,0.0978613297144572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,65535,0.18178133169809976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,1,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,1,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,1,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,1,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,3,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,3,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,3,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,7,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,7,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,7,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,15,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,15,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,15,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,float16,131071,0.3489919900894165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,15,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,8,4,128,0,1,float16,fp8,131071,0.17881067593892416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,31,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,31,0.010656000425418219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,31,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,63,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,63,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,63,0.011717333147923151
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,127,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,127,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,127,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,127,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,255,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,255,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,255,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,255,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,511,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,511,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,511,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,511,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,1023,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,1023,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,1023,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,1023,0.011722666521867117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,2047,0.015647999942302704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,2047,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,2047,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,2047,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,4095,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,4095,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,4095,0.013552000125249227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,4095,0.01716800034046173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,8191,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,8191,0.0191040001809597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,8191,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,8191,0.019253333409627277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,16383,0.013530666629473368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,16383,0.02364266663789749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,16383,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,16383,0.023370665808518726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,32767,0.038618666430314384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,32767,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,32767,0.013455999394257864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,32767,0.03154666721820831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,65535,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,65535,0.060720001657803856
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,65535,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,65535,0.05267733335494995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,float16,131071,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,float16,131071,0.10342933734258015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,1,0.012122667084137598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,128,1,float16,fp8,131071,0.015509333461523056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,1,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,1,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,1,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,3,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,3,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,3,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,3,0.011834666132926941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,7,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,7,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,7,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,64,0,1,float16,fp8,131071,0.08932266632715861
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,15,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,15,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,15,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,15,0.011701333026091257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,31,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,31,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,31,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,31,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,63,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,63,0.010575999816258749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,63,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,127,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,127,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,127,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,127,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,255,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,255,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,255,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,255,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,511,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,511,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,511,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,511,0.012698666503032049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,1023,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,1023,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,1023,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,2047,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,2047,0.017194667210181553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,2047,0.014090667168299357
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,2047,0.01691199963291486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,4095,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,4095,0.01934933289885521
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,4095,0.013882666826248169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,4095,0.019482667247454327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,8191,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,8191,0.02548266698916753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,8191,0.023247999449570973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,8191,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,16383,0.013514666507641474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,16383,0.03958400090535482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,16383,0.014010666559139887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,16383,0.032032000521818794
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,32767,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,32767,0.05966933568318685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,32767,0.013568000247081121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,32767,0.05225066840648651
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,65535,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,65535,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,65535,0.10078932841618855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,65535,0.08711466193199158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,float16,131071,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,1,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,float16,131071,0.18460800250371298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,1,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,1,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,3,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,1,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,3,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,3,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,3,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,7,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,7,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,7,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,15,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,7,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,15,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,15,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,15,0.011776000261306763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,128,1,float16,fp8,131071,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,31,0.010527999450763067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,64,0,1,float16,fp8,131071,0.15892799695332846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,31,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,31,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,31,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,63,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,63,0.009749333063761393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,63,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,127,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,63,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,127,0.009994666402538618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,127,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,127,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,255,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,255,0.010069333637754122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,255,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,511,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,511,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,511,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,511,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,1023,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,1023,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,1023,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,2047,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,2047,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,2047,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,2047,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,4095,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,4095,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,4095,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,4095,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,8191,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,8191,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,8191,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,8191,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,16383,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,16383,0.01959466685851415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,16383,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,16383,0.01841066653529803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,32767,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,32767,0.029178666571776073
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,32767,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,32767,0.02569599946339925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,65535,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,65535,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,65535,0.033573334415753685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,65535,0.029301332930723827
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,float16,131071,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,1,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,float16,131071,0.03349333256483078
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,128,1,float16,fp8,131071,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,64,0,1,float16,fp8,131071,0.031498665610949196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,1,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,1,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,1,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,3,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,3,0.010575999816258749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,3,0.010725333044926325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,3,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,7,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,7,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,7,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,7,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,15,0.010330666477481524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,15,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,15,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,31,0.010805333654085795
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,31,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,31,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,31,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,63,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,63,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,63,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,63,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,127,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,127,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,127,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,127,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,255,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,255,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,255,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,255,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,511,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,511,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,511,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,511,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,1023,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,1023,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,1023,0.011653333902359009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,1023,0.013349333157142004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,2047,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,2047,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,2047,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,2047,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,4095,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,4095,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,4095,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,4095,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,8191,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,8191,0.015509333461523056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,8191,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,8191,0.011701333026091257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,16383,0.01163200040658315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,16383,0.018933333456516266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,16383,0.01163200040658315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,16383,0.01974933346112569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,32767,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,32767,0.02203733225663503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,32767,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,32767,0.023370665808518726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,65535,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,65535,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,65535,0.023472001155217487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,65535,0.02179199953873952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,float16,131071,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,float16,131071,0.02805333336194356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,1,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,1,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,1,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,1,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,3,0.010538666198650995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,3,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,3,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,3,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,7,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,128,1,float16,fp8,131071,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,7,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,7,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,7,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,15,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,15,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,15,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,15,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,31,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,31,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,31,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,31,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,64,0,1,float16,fp8,131071,0.027450665831565857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,63,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,63,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,63,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,63,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,127,0.010677333921194077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,127,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,127,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,255,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,255,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,255,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,511,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,511,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,511,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,511,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,1023,0.01055466632048289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,1023,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,1023,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,1023,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,2047,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,2047,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,2047,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,2047,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,4095,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,4095,0.014101333916187286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,4095,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,4095,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,8191,0.012709333250919977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,8191,0.016917333006858826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,8191,0.011616000284751257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,8191,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,16383,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,16383,0.021231998999913532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,16383,0.01923199991385142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,16383,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,32767,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,32767,0.023423999547958374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,32767,0.011706666400035223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,32767,0.023306667804718018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,65535,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,65535,0.025413334369659424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,65535,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,65535,0.023333333432674408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,float16,131071,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,float16,131071,0.027621333797772724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,128,1,float16,fp8,131071,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,1,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,1,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,1,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,1,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,3,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,3,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,3,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,3,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,7,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,7,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,7,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,7,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,15,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,15,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,15,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,15,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,64,0,1,float16,fp8,131071,0.027295999228954315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,31,0.010602666685978571
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,31,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,31,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,31,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,63,0.010378666842977205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,63,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,63,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,63,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,127,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,127,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,127,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,127,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,255,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,255,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,255,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,511,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,511,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,511,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,511,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,1023,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,1023,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,1023,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,1023,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,2047,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,2047,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,2047,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,2047,0.01246400053302447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,4095,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,4095,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,4095,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,4095,0.01267733300725619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,8191,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,8191,0.017136000096797943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,8191,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,8191,0.016666666915019352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,16383,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,16383,0.019285333653291065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,16383,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,16383,0.01695466662446658
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,32767,0.021055998901526134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,32767,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,32767,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,32767,0.019424000134070713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,65535,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,65535,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,65535,0.023178666830062866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,65535,0.021295999487241108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,float16,131071,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,1,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,float16,131071,0.029877332349618275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,128,1,float16,fp8,131071,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,64,0,1,float16,fp8,131071,0.027503999571005504
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,1,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,1,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,1,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,3,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,3,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,3,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,3,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,7,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,7,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,7,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,15,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,15,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,15,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,15,0.012383999923865
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,31,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,31,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,31,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,63,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,63,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,63,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,63,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,127,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,127,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,127,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,127,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,255,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,255,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,511,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,511,0.011690666278203329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,511,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,511,0.012053333222866058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,1023,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,1023,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,1023,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,1023,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,2047,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,2047,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,2047,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,2047,0.017407999684413273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,4095,0.01947733387351036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,4095,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,4095,0.019130667050679524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,4095,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,8191,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,8191,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,8191,0.013866666704416275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,8191,0.023775999744733173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,16383,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,16383,0.039674667020638786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,16383,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,16383,0.031317333380381264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,32767,0.013418667018413544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,32767,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,32767,0.06062933305899302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,32767,0.052560001611709595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,float16,65535,0.013754667093356451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,float16,65535,0.10120532910029094
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,128,1,float16,fp8,65535,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,1,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,1,0.011711999773979187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,1,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,1,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,3,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,3,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,3,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,3,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,7,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,7,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,64,0,1,float16,fp8,65535,0.0869599978129069
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,7,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,7,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,15,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,15,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,15,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,31,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,31,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,63,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,63,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,63,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,63,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,127,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,127,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,127,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,127,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,255,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,255,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,255,0.011866666376590729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,511,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,511,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,511,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,511,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,1023,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,1023,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,1023,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,1023,0.012698666503032049
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,2047,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,2047,0.019082666685183842
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,2047,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,2047,0.01727466657757759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,4095,0.013850666582584381
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,4095,0.02382933348417282
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,4095,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,4095,0.021269333859284718
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,8191,0.013658666362365087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,8191,0.013951999445756277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,8191,0.038032000263532005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,8191,0.03154666721820831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,16383,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,16383,0.0591893345117569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,16383,0.0136266661187013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,16383,0.05213333169619242
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,32767,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,32767,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,32767,0.10084799925486247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,32767,0.08691733082135518
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,float16,65535,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,1,0.010656000425418219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,1,0.010575999816258749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,1,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,float16,65535,0.1845546762148539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,1,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,3,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,3,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,3,0.011861333002646765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,7,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,7,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,7,0.010533332824707031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,128,1,float16,fp8,65535,0.014175999909639359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,15,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,7,0.011706666400035223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,15,0.010703999549150467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,15,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,15,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,31,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,31,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,31,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,31,0.012293333808581034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,63,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,63,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,64,0,1,float16,fp8,65535,0.1554080049196879
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,63,0.011887999872366587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,63,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,127,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,127,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,127,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,255,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,255,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,255,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,255,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,511,0.010586666564146677
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,511,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,511,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,511,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,1023,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,1023,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,1023,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,1023,0.012442667037248611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,2047,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,2047,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,2047,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,2047,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,4095,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,4095,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,4095,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,4095,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,8191,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,8191,0.017583999782800674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,8191,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,8191,0.019120000302791595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,16383,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,16383,0.01966399947802226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,16383,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,16383,0.019674666225910187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,32767,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,32767,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,32767,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,32767,0.021136000752449036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,65535,0.012719999998807907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,65535,0.023210667073726654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,65535,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,65535,0.021509334444999695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,float16,131071,0.014906667172908783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,1,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,1,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,float16,131071,0.029359998802344005
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,128,1,float16,fp8,131071,0.014842666685581207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,1,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,64,0,1,float16,fp8,131071,0.027610667049884796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,1,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,3,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,3,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,3,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,3,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,7,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,7,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,7,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,15,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,15,0.010634666929642359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,15,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,15,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,31,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,31,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,31,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,31,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,63,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,63,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,63,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,127,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,127,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,127,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,127,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,255,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,255,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,255,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,255,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,511,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,511,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,511,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,511,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,1023,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,1023,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,1023,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,1023,0.01228800043463707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,2047,0.012629333883523941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,2047,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,2047,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,2047,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,4095,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,4095,0.017018667111794155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,4095,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,4095,0.017029333859682083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,8191,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,8191,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,8191,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,8191,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,16383,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,16383,0.018922666708628338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,16383,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,16383,0.018768000106016796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,32767,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,32767,0.02146666745344798
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,32767,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,32767,0.019776000330845516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,65535,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,65535,0.02510933329661687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,65535,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,65535,0.02359466751416524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,float16,131071,0.015360000232855478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,1,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,float16,131071,0.042597333590189614
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,128,1,float16,fp8,131071,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,1,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,1,0.012437333663304647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,64,0,1,float16,fp8,131071,0.03348266581694285
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,1,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,3,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,3,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,3,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,3,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,7,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,7,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,7,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,7,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,15,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,15,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,31,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,31,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,31,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,31,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,63,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,63,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,63,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,63,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,127,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,127,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,127,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,127,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,255,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,255,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,255,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,511,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,511,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,511,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,511,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,1023,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,1023,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,1023,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,1023,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,2047,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,2047,0.019391999890406925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,2047,0.014736000448465347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,2047,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,4095,0.01482133318980535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,4095,0.02378133436044057
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,4095,0.014762666076421738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,4095,0.02359466751416524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,8191,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,8191,0.038586666186650596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,8191,0.01532799998919169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,8191,0.031930667658646904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,16383,0.014842666685581207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,16383,0.05952533086140951
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,16383,0.015530666957298914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,16383,0.05187733471393585
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,float16,32767,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,128,1,float16,fp8,32767,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,float16,32767,0.10170666376749675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,1,0.013866666704416275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,1,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,1,0.01368533323208491
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,3,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,1,0.013994666437307993
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,3,0.012202666451533636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,3,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,3,0.013621332744757334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,7,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,7,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,7,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,15,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,7,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,15,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,15,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,31,0.012693333129088083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,31,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,31,0.011621333658695221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,31,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,63,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,63,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,64,0,1,float16,fp8,32767,0.08699733018875122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,63,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,63,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,127,0.011616000284751257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,127,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,127,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,127,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,255,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,255,0.011786667009194693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,255,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,255,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,511,0.012608000387748083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,511,0.013552000125249227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,511,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,511,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,1023,0.011690666278203329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,1023,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,1023,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,1023,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,2047,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,2047,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,2047,0.025381334125995636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,2047,0.023050665855407715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,4095,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,4095,0.040031999349594116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,4095,0.01534933348496755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,4095,0.03236266722281774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,8191,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,8191,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,8191,0.061946665247281395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,8191,0.05379199981689453
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,16383,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,16383,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,16383,0.10335999727249146
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,16383,0.08829866846402486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,float16,32767,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,1,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,128,1,float16,fp8,32767,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,float16,32767,0.18703466653823853
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,64,0,1,float16,fp8,32767,0.15707199772198996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,1,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,1,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,1,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,3,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,3,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,3,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,3,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,7,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,7,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,7,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,7,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,15,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,15,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,15,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,15,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,31,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,31,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,31,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,31,0.01267733300725619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,63,0.011749333391586939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,63,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,63,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,127,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,127,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,127,0.012741333494583765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,127,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,255,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,255,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,255,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,255,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,511,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,511,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,511,0.01392000044385592
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,511,0.014597332725922266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,1023,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,1023,0.01720533271630605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,1023,0.01746133342385292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,1023,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,2047,0.015743999431530636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,2047,0.025301332275072735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,2047,0.01544533297419548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,2047,0.023605334262053173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,4095,0.014981333166360855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,4095,0.040474665661652885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,4095,0.015882667154073715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,4095,0.031658666829268135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,float16,8191,0.015423999478419622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,128,1,float16,fp8,8191,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,float16,8191,0.06149866680304209
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,1,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,1,0.014005333185195923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,1,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,1,0.016165333489576977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,3,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,3,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,64,0,1,float16,fp8,8191,0.05271466573079427
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,3,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,3,0.015610666324694952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,7,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,7,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,7,0.014954666296641031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,7,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,15,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,15,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,15,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,15,0.015077333897352219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,31,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,31,0.01370666672786077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,31,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,31,0.01488000030318896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,63,0.014730667074521383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,63,0.014975999792416891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,63,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,63,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,127,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,127,0.014783999572197596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,127,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,127,0.014853333433469137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,255,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,255,0.014746667196353277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,255,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,255,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,511,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,511,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,511,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,511,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,1023,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,1023,0.023082666099071503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,1023,0.0138026662170887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,1023,0.021301334102948506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,2047,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,2047,0.04051200052102407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,2047,0.017024000485738117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,2047,0.03198933353026708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,4095,0.017184000462293625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,4095,0.0625439981619517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,4095,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,4095,0.05454400181770325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,float16,8191,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,float16,8191,0.10436266660690308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,128,1,float16,fp8,8191,0.0170666662355264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,64,0,1,float16,fp8,8191,0.08734933535257976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,1,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,1,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,1,0.01381333296497663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,1,0.014752000570297241
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,3,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,3,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,3,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,3,0.013770667215188345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,7,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,7,0.014725333700577417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,7,0.013690666606028875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,7,0.014837333311637243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,15,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,15,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,15,0.014080000420411428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,15,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,31,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,31,0.01488000030318896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,31,0.01381333296497663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,31,0.014917333920796713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,63,0.01469333345691363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,63,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,63,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,63,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,127,0.013530666629473368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,127,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,127,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,127,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,255,0.016634666671355564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,255,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,255,0.014885333677132925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,255,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,511,0.013978666315476099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,511,0.01718933383623759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,511,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,511,0.01711999997496605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,1023,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,1023,0.014730667074521383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,1023,0.02370133250951767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,1023,0.022042666872342426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,2047,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,2047,0.04040000090996424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,2047,0.032458665470282234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,2047,0.017690667261679966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,float16,4095,0.017925333231687546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,float16,4095,0.062319998939832054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,1,0.017466666797796886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,0,1,float16,fp8,4095,0.05340800185998281
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,64,128,1,float16,fp8,4095,0.018005333840847015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,1,0.01749333366751671
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,1,0.017680000513792038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,1,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,3,0.01752000053723653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,3,0.01762666677435239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,3,0.017743999759356182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,3,0.01756799966096878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,7,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,7,0.017407999684413273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,7,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,7,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,15,0.01703466723362605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,15,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,15,0.017450666675964992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,15,0.017749333133300144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,31,0.017173333714405697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,31,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,31,0.017759999881188076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,31,0.017792000124851864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,63,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,63,0.0174346665541331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,63,0.017711999515692394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,63,0.017653333644072216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,127,0.017349333812793095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,127,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,127,0.017903999735911686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,127,0.01743999992807706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,255,0.017349333812793095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,255,0.017573333034912746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,255,0.017765333255132038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,255,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,511,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,511,0.023376000424226124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,511,0.017504000415404636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,511,0.021744000415007275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,1023,0.01718933383623759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,1023,0.017530667285124462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,1023,0.034117333590984344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,1023,0.02923733244339625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,2047,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,2047,0.01956266661485036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,2047,0.05792533357938131
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,2047,0.05021866659323374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,float16,4095,0.020954666038354237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,1,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,1,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,1,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,128,1,float16,fp8,4095,0.02362666775782903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,float16,4095,0.09650133053461711
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,64,0,1,float16,fp8,4095,0.08135466774304707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,3,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,3,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,3,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,7,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,7,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,7,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,7,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,15,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,15,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,15,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,31,0.010687999427318573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,15,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,31,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,31,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,63,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,63,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,63,0.011578666667143503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,63,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,127,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,127,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,127,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,127,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,255,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,255,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,255,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,511,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,511,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,511,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,511,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,1023,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,1023,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,1023,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,2047,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,2047,0.01357866699496905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,2047,0.013679999858140945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,2047,0.015840000162522
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,4095,0.013503999759753546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,4095,0.01724799970785777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,4095,0.013642666240533194
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,4095,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,8191,0.013631999492645264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,8191,0.01966933285196622
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,8191,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,8191,0.017504000415404636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,16383,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,16383,0.019066666563351948
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,16383,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,16383,0.019343999524911244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,32767,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,32767,0.021295999487241108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,32767,0.013573333621025085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,32767,0.020341333001852036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,65535,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,65535,0.025989333788553875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,65535,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,65535,0.025583999852339428
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,float16,131071,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,1,0.010735999792814255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,float16,131071,0.042650664846102394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,1,0.0106133334338665
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,1,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,128,1,float16,fp8,131071,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,1,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,3,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,3,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,3,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,3,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,64,0,1,float16,fp8,131071,0.033439998825391136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,7,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,7,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,7,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,7,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,15,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,15,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,15,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,15,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,31,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,31,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,31,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,31,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,63,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,63,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,63,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,63,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,127,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,127,0.010640000303586325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,127,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,255,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,255,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,255,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,255,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,511,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,511,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,511,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,511,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,1023,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,1023,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,1023,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,1023,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,2047,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,2047,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,2047,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,2047,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,4095,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,4095,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,4095,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,4095,0.017071999609470367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,8191,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,8191,0.018496000518401463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,8191,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,8191,0.017759999881188076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,16383,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,16383,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,16383,0.021055998901526134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,16383,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,32767,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,32767,0.02548266698916753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,32767,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,32767,0.023818666736284893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,65535,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,65535,0.0402453343073527
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,65535,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,65535,0.03190399954716364
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,float16,131071,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,float16,131071,0.06046399970849355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,128,1,float16,fp8,131071,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,1,0.017498667041460674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,1,0.018170667191346485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,1,0.017642666896184284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,1,0.017466666797796886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,3,0.017338667064905167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,64,0,1,float16,fp8,131071,0.05412266651789347
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,3,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,3,0.01756799966096878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,3,0.017711999515692394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,7,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,7,0.017573333034912746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,7,0.017637333522240322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,7,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,15,0.017903999735911686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,15,0.017386666188637417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,15,0.017237332959969837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,15,0.017360000560681026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,31,0.017504000415404636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,31,0.01748266691962878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,31,0.017722666263580322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,31,0.017290666699409485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,63,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,63,0.017808000246683758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,63,0.01747200017174085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,63,0.017466666797796886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,127,0.01727466657757759
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,127,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,127,0.017653333644072216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,127,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,255,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,255,0.017680000513792038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,255,0.017664000391960144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,255,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,511,0.018122666825850803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,511,0.023237332701683044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,511,0.01773333301146825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,511,0.02295999974012375
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,1023,0.019296000401178997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,1023,0.03411199897527695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,1023,0.01783466711640358
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,1023,0.02942399928967158
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,float16,2047,0.021776000658671062
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,float16,2047,0.05875733494758606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,128,1,float16,fp8,2047,0.02130666623512904
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,64,0,1,float16,fp8,2047,0.05212800204753876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,1,0.025274666647116344
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,1,0.02550400048494339
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,1,0.025040000677108765
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,1,0.025557334224383037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,3,0.025648000339667004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,3,0.026975999275843304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,3,0.025072000920772552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,3,0.02568000058333079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,7,0.02536533276240031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,7,0.025434667865435284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,7,0.02348266790310542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,7,0.025733334322770435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,15,0.025290665527184803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,15,0.025258667767047882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,15,0.023434666295846302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,15,0.02384000023206075
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,31,0.02548266698916753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,31,0.02526933451493581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,31,0.02350933353106181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,31,0.023941333095232647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,63,0.025631998976071674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,63,0.025253333151340485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,63,0.025242666403452556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,63,0.0240639994541804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,127,0.02362666775782903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,127,0.026026666164398193
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,127,0.025029333929220837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,127,0.023743999501069386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,255,0.02566933383544286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,255,0.02514133354028066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,255,0.026165333886941273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,255,0.025311999022960663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,511,0.02566933383544286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,511,0.03770666569471359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,511,0.02532266577084859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,511,0.0341333324710528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,1023,0.025536000728607178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,1023,0.05576533575852712
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,1023,0.0277813325325648
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,1023,0.05060266455014547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,float16,2047,0.02974933385848999
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,float16,2047,0.09733866651852925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,128,1,float16,fp8,2047,0.02810666710138321
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,1,0.025216000775496166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,1,0.026208000878492992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,64,0,1,float16,fp8,2047,0.08561600248018901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,1,0.025434667865435284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,1,0.025813333690166473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,3,0.025706666211287182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,3,0.025333332518736523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,3,0.025786665578683216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,3,0.025994665920734406
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,7,0.02532800038655599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,7,0.02516799916823705
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,7,0.025333332518736523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,7,0.025850666066010792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,15,0.025263999899228413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,15,0.02554133286078771
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,15,0.02535466601451238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,15,0.025360000630219776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,31,0.025077333052953083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,31,0.025072000920772552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,31,0.02513599892457326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,31,0.025360000630219776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,63,0.02537599951028824
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,63,0.02518933266401291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,63,0.025248001019159954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,63,0.025600001215934753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,127,0.02497066557407379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,127,0.02496533344189326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,127,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,127,0.025429333249727886
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,255,0.027327999472618103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,255,0.0240639994541804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,255,0.02534399926662445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,255,0.025557334224383037
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,511,0.026975999275843304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,511,0.0366239994764328
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,511,0.02610666553179423
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,511,0.03418133407831192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,float16,1023,0.025216000775496166
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,float16,1023,0.05479466418425242
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,128,1,float16,fp8,1023,0.02630399912595749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,64,0,1,float16,fp8,1023,0.050810664892196655
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,1,0.03952533255020777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,1,0.03999999910593033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,1,0.03783999880154928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,1,0.037530665596326195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,3,0.0399893323580424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,3,0.0397119993964831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,3,0.037871999045213066
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,3,0.03765333443880081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,7,0.03990933299064636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,7,0.039450667798519135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,7,0.03794133414824804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,7,0.03781333317359289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,15,0.03935466706752777
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,15,0.03973866750796636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,15,0.03781333317359289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,15,0.037690666814645134
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,31,0.039450667798519135
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,31,0.03999999910593033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,63,0.03920533259709676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,63,0.03945599993069967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,63,0.038165333370367684
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,63,0.03803733239571253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,127,0.039503999054431915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,127,0.03894400099913279
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,127,0.037802666425704956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,31,0.03830400109291077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,31,0.03895466774702072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,127,0.03753600021203359
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,255,0.042208001017570496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,255,0.042352000872294106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,255,0.040106666584809623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,511,0.04224533339341482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,511,0.06052800019582113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,511,0.03915199885765711
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,511,0.05690666536490122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,float16,1023,0.04178666571776072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,float16,1023,0.09623466928799947
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,1,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,128,1,float16,fp8,1023,0.03985599925120672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,1,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,1,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,3,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,3,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,3,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,1023,0.08597333232561748
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,7,0.01055466632048289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,7,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,7,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,15,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,15,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,64,0,1,float16,fp8,255,0.03807999938726425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,15,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,31,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,31,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,31,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,31,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,63,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,63,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,63,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,63,0.011653333902359009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,127,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,127,0.0107893335322539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,127,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,127,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,255,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,255,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,255,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,511,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,255,0.011722666521867117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,511,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,511,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,511,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,1023,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,1023,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,1023,0.011786667009194693
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,2047,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,1023,0.013679999858140945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,2047,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,2047,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,2047,0.015568000574906668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,4095,0.016970666746298473
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,4095,0.01340266689658165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,4095,0.013541333377361298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,4095,0.01720533271630605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,8191,0.013503999759753546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,8191,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,8191,0.01800000046690305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,8191,0.01859733338157336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,16383,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,16383,0.022053333620230358
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,16383,0.013770667215188345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,16383,0.019359999646743137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,32767,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,32767,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,32767,0.02565866708755493
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,32767,0.02372266600529353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,65535,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,65535,0.04015466570854187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,65535,0.013770667215188345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,65535,0.03147733211517334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,float16,131071,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,1,0.011850666254758835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,1,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,float16,131071,0.061280002196629844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,1,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,128,1,float16,fp8,131071,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,64,0,1,float16,fp8,131071,0.05329599976539612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,3,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,1,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,3,0.010677333921194077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,3,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,3,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,7,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,7,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,7,0.011834666132926941
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,7,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,15,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,15,0.010698666175206503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,15,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,15,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,31,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,31,0.010570666442314783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,31,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,31,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,63,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,63,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,63,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,63,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,127,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,127,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,127,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,127,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,255,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,255,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,255,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,255,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,511,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,511,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,511,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,511,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,1023,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,1023,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,1023,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,1023,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,2047,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,2047,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,2047,0.01349866638580958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,2047,0.014655999839305878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,4095,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,4095,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,4095,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,4095,0.01540800059835116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,8191,0.01905599981546402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,8191,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,8191,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,8191,0.017338667064905167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,16383,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,16383,0.023770667612552643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,16383,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,16383,0.0216799999276797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,32767,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,32767,0.039594667653242745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,32767,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,32767,0.0315733328461647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,65535,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,65535,0.05959466596444448
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,65535,0.013557333499193192
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,65535,0.0524479995171229
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,float16,131071,0.015626666446526844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,1,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,float16,131071,0.10220266381899516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,1,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,3,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,3,0.011695999652147293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,7,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,7,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,15,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,15,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,128,1,float16,fp8,131071,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,31,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,31,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,63,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,127,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,63,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,64,0,1,float16,fp8,131071,0.08922666311264038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,127,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,511,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,511,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,1023,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,1023,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,2047,0.015087999403476715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,2047,0.016976000120242436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,4095,0.017781333376963932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,4095,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,8191,0.019205333044131596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,8191,0.02142400046189626
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,16383,0.0397119993964831
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,16383,0.025311999022960663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,32767,0.0396373321612676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,32767,0.06144000093142191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,65535,0.06037333110968272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,65535,0.1023466686407725
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,1,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,3,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,1,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,3,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,7,0.011877333124478659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,7,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,15,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,15,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,31,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,31,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,63,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,127,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,63,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,127,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,255,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,255,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,511,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,511,0.013552000125249227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,1023,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,1023,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,2047,0.018778666853904724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,fp8,131071,0.10191466410954793
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,1,128,0,1,float16,float16,131071,0.18442134062449136
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,2047,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,4095,0.021546666820844013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,4095,0.019578666736682255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,8191,0.023408000667889912
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,8191,0.03985599925120672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,16383,0.059893334905306496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,16383,0.04044266790151596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,32767,0.10126933455467224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,32767,0.05950400233268738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,65535,0.18320000171661377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,65535,0.09822932879130046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,1,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,3,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,3,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,7,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,7,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,15,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,15,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,31,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,31,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,63,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,63,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,127,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,127,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,511,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,511,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,1023,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,float16,131071,0.34718934694925946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,1023,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,4,2,128,0,1,float16,fp8,131071,0.17987199624379477
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,2047,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,2047,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,4095,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,4095,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,8191,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,8191,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,16383,0.01953599974513054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,16383,0.01985599969824155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,32767,0.029701332251230877
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,32767,0.026122666895389557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,65535,0.03258133431275686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,65535,0.033045334120591484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,1,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,1,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,3,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,float16,131071,0.035887998839219414
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,3,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,7,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,7,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,15,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,31,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,15,0.011744000017642975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,31,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,63,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,63,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,1,128,0,1,float16,fp8,131071,0.035536001125971474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,127,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,127,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,255,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,511,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,255,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,511,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,1023,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,1023,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,2047,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,2047,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,4095,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,4095,0.013434667140245438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,8191,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,8191,0.0180479995906353
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,16383,0.01995733380317688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,16383,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,32767,0.023738667368888855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,32767,0.02146133283774058
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,65535,0.02624000112215678
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,1,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,65535,0.024656000236670177
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,1,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,3,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,3,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,7,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,7,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,15,0.01211200033624967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,float16,131071,0.03179199993610382
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,31,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,31,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,4,2,128,0,1,float16,fp8,131071,0.02789866675933202
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,127,0.010725333044926325
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,255,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,127,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,255,0.010618666807810465
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,511,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,511,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,1023,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,1023,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,2047,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,2047,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,4095,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,4095,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,8191,0.01752000053723653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,8191,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,16383,0.021402666966120403
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,16383,0.02089066555102666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,32767,0.02459733436505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,32767,0.024101334313551586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,65535,0.025989333788553875
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,65535,0.025642665723959606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,1,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,1,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,3,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,3,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,7,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,7,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,15,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,15,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,31,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,31,0.011600000162919363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,63,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,float16,131071,0.03239466746648153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,63,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,127,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,1,128,0,1,float16,fp8,131071,0.029482667644818623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,255,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,255,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,511,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,511,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,1023,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,1023,0.011754666765530905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,2047,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,2047,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,4095,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,4095,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,8191,0.01721599946419398
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,8191,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,16383,0.02109333376089732
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,16383,0.01985599969824155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,32767,0.02367999901374181
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,32767,0.021701333423455555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,65535,0.025744001070658367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,65535,0.022853332261244457
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,1,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,1,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,3,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,float16,131071,0.04530133306980133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,3,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,7,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,7,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,15,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,4,2,128,0,1,float16,fp8,131071,0.02762666592995326
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,31,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,31,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,63,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,127,0.010762666662534079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,63,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,127,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,255,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,511,0.011781333635250727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,511,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,1023,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,1023,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,2047,0.019152000546455383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,2047,0.017562666287024815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,4095,0.021418665846188862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,4095,0.019498666127522785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,8191,0.04095466683308283
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,8191,0.025759999950726826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,16383,0.06241066753864288
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,16383,0.03996799886226654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,32767,0.1027733286221822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,32767,0.059658666451772056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,1,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,1,0.01191466674208641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,3,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,7,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,3,0.01184533288081487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,7,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,15,0.012442667037248611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,15,0.01488000030318896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,31,0.012560000022252401
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,31,0.01228800043463707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,63,0.011968000481526056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,63,0.011770666887362799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,127,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,255,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,float16,65535,0.18413333098093668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,127,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,1,128,0,1,float16,fp8,65535,0.09929066896438599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,255,0.011994666109482447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,511,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,511,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,1023,0.014074667046467463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,1023,0.013477332890033722
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,2047,0.021482666333516438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,2047,0.019333332777023315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,4095,0.03977599988381068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,4095,0.023631999890009563
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,8191,0.039861333866914116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,8191,0.060421332716941833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,16383,0.0581279993057251
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,16383,0.10021332899729411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,32767,0.18085867166519165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,32767,0.09795733292897542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,1,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,1,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,3,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,3,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,7,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,7,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,15,0.010575999816258749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,15,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,63,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,31,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,63,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,127,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,127,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,255,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,255,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,511,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,fp8,65535,0.17717333634694418
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,4,2,128,0,1,float16,float16,65535,0.344160000483195
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,511,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,1023,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,1023,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,2047,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,2047,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,4095,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,4095,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,8191,0.019610666980346043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,8191,0.019381333142518997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,16383,0.021183999876181286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,16383,0.021082667013009388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,32767,0.023893333971500397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,32767,0.02181333303451538
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,65535,0.027717334528764088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,65535,0.023573334018389385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,1,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,1,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,3,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,3,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,7,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,float16,131071,0.0479360024134318
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,7,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,15,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,15,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,31,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,31,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,1,128,0,1,float16,fp8,131071,0.031290667752424874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,63,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,63,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,127,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,255,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,255,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,511,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,1023,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,1023,0.011781333635250727
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,511,0.011877333124478659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,2047,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,2047,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,4095,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,4095,0.01746133342385292
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,8191,0.019871999820073444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,8191,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,16383,0.021727999051411945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,16383,0.01964266722400983
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,32767,0.023941333095232647
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,32767,0.021354667842388153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,65535,0.04200000067551931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,65535,0.025568000972270966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,1,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,1,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,3,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,3,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,7,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,15,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,7,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,15,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,float16,131071,0.06355733176072438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,4,2,128,0,1,float16,fp8,131071,0.04385066529115041
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,31,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,31,0.011839999506870905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,63,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,127,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,63,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,127,0.012458667159080505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,255,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,255,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,511,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,511,0.01259200026591619
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,1023,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,1023,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,2047,0.021125334004561108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,2047,0.019130667050679524
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,4095,0.0396373321612676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,4095,0.02421333392461141
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,8191,0.039317332208156586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,8191,0.06121600170930227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,16383,0.10115733742713928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,16383,0.05946666498978933
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,1,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,1,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,3,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,3,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,7,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,7,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,15,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,15,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,31,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,31,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,float16,32767,0.18202133973439535
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,63,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,63,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,1,128,0,1,float16,fp8,32767,0.0990666647752126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,127,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,127,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,255,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,255,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,511,0.01488000030318896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,511,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,1023,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,1023,0.01756799966096878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,2047,0.03984533250331879
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,2047,0.02518933266401291
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,4095,0.06075199941794077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,4095,0.04087999959786733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,8191,0.1034346620241801
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,8191,0.061343997716903687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,16383,0.18232532342274985
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,16383,0.10274133086204529
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,1,0.013594667116800943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,1,0.01358933374285698
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,float16,32767,0.34770135084788006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,3,0.01357866699496905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,4,2,128,0,1,float16,fp8,32767,0.18318933248519897
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,3,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,7,0.013728000223636627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,7,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,15,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,15,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,31,0.013669333110253016
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,31,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,63,0.013477332890033722
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,63,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,127,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,255,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,255,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,127,0.014074667046467463
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,511,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,511,0.013861333330472311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,1023,0.017637333522240322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,2047,0.04014399896065394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,1023,0.017952000101407368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,2047,0.025498665869235992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,4095,0.061199997862180076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,4095,0.04113066693147024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,1,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,1,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,3,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,float16,8191,0.10316266616185506
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,3,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,1,128,0,1,float16,fp8,8191,0.061808000008265175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,7,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,7,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,15,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,15,0.014842666685581207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,31,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,31,0.015781333049138386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,63,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,63,0.015109332899252573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,127,0.015333333363135656
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,127,0.015589332828919092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,255,0.015471999843915304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,255,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,511,0.01923199991385142
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,511,0.017722666263580322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,1023,0.023205332458019257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,1023,0.03589333345492681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,2047,0.06202666461467743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,2047,0.0413973331451416
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,4095,0.10354666908582051
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,4095,0.06263466676076253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,1,0.015669333438078564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,1,0.015306666493415833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,fp8,8191,0.103301336367925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,4,2,128,0,1,float16,float16,8191,0.18515199422836304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,3,0.015008000036080679
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,3,0.015557333827018738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,7,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,7,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,15,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,15,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,31,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,31,0.01532799998919169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,63,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,63,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,127,0.014890667051076889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,127,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,255,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,255,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,511,0.01933866615096728
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,511,0.01762666677435239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,1023,0.035616000493367515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,1023,0.023120000958442688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,2047,0.06158400078614553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,2047,0.041162667175134025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,1,0.01916266605257988
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,1,0.019402666638294857
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,float16,4095,0.10394133130709331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,3,0.019071999937295914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,1,128,0,1,float16,fp8,4095,0.06284800171852112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,3,0.01926933353145917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,7,0.018901333212852478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,7,0.019541333119074505
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,15,0.01747200017174085
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,15,0.018917333334684372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,31,0.019359999646743137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,31,0.019120000302791595
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,63,0.019343999524911244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,63,0.01911466692884763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,127,0.020021333048741024
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,127,0.017866666118303936
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,255,0.019679999599854153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,255,0.017743999759356182
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,511,0.03336533407370249
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,511,0.02319466571013133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,1023,0.0535093347231547
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,1023,0.034485332667827606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,2047,0.09834133585294087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,2047,0.05889600018660227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,1,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,1,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,float16,4095,0.17371733983357748
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,3,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,4,2,128,0,1,float16,fp8,4095,0.09770133097966512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,3,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,7,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,7,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,15,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,15,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,31,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,31,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,63,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,63,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,127,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,127,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,255,0.010650667051474253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,255,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,511,0.011717333147923151
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,511,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,1023,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,1023,0.012527999778588613
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,2047,0.014815999815861383
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,2047,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,4095,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,4095,0.017008000363906223
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,8191,0.019343999524911244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,8191,0.01905599981546402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,16383,0.021130666136741638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,16383,0.020080000162124634
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,32767,0.02535466601451238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,32767,0.021498667697111767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,65535,0.04185600082079569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,1,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,65535,0.025968000292778015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,3,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,7,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,7,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,15,0.010741333166758219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,31,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,15,0.011898666620254517
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,31,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,63,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,63,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,127,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,127,0.010597333312034607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,fp8,131071,0.04294399917125702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,1,128,0,1,float16,float16,131071,0.06331199904282887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,255,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,511,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,511,0.01268799975514412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,1023,0.014666666587193808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,1023,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,2047,0.01703466723362605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,2047,0.01693333312869072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,4095,0.01710933322707812
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,4095,0.017237332959969837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,8191,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,8191,0.0191040001809597
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,16383,0.02126399924357732
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,16383,0.023397333920001984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,32767,0.04204800228277842
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,32767,0.025568000972270966
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,65535,0.06001600126425425
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,65535,0.04011733333269755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,1,0.019381333142518997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,1,0.019013332823912304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,float16,131071,0.1027786632378896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,4,2,128,0,1,float16,fp8,131071,0.06076266864935557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,3,0.019343999524911244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,3,0.01905599981546402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,7,0.019386666516462963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,7,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,15,0.018992000569899876
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,15,0.01922133316596349
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,31,0.019685332973798115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,31,0.01882133384545644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,63,0.019546666493018467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,63,0.0174346665541331
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,127,0.019194666296243668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,127,0.018895999838908512
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,255,0.01918399954835574
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,255,0.019013332823912304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,511,0.03346133232116699
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,511,0.023397333920001984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,1023,0.05336533486843109
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,1023,0.033530667424201965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,float16,2047,0.0981173316637675
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,1,0.02754666656255722
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,1,128,0,1,float16,fp8,2047,0.05872533222039541
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,1,0.02571200082699458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,3,0.028010666370391846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,3,0.026416001220544178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,7,0.027615999182065327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,7,0.026047999660174053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,15,0.02568000058333079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,15,0.029338667790095013
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,31,0.02824000020821889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,31,0.025663999219735462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,63,0.029440000653266907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,127,0.02777066578467687
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,127,0.025439999997615814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,63,0.026517334083716076
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,255,0.03422933320204417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,255,0.026047999660174053
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,511,0.05379733443260193
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,511,0.038245332737763725
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,1023,0.09249066313107808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,1023,0.055770665407180786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,float16,2047,0.1751413345336914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,1,0.029722665747006733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,4,2,128,0,1,float16,fp8,2047,0.09887466828028361
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,1,0.02646933247645696
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,3,0.029317334294319153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,3,0.02585600068171819
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,7,0.028677334388097126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,7,0.025983999172846477
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,15,0.02938666691382726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,15,0.02607999990383784
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,31,0.028762665887673695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,31,0.025957333544890087
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,63,0.02959999938805898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,63,0.0262773334980011
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,127,0.02775466690460841
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,127,0.02573866645495097
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,255,0.0334346666932106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,255,0.025941332181294758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,511,0.05470933516820272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,511,0.038202665746212006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,float16,1023,0.09274133046468098
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,1,0.04611733555793762
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,1,0.040250666439533234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,1,128,0,1,float16,fp8,1023,0.05565866827964783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,3,0.046037331223487854
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,3,0.03999999910593033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,7,0.045935998360315956
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,7,0.04027199993530909
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,15,0.04593066871166229
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,15,0.0400693342089653
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,31,0.04580266773700714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,31,0.04186666508515676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,63,0.046341334780057274
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,63,0.040048000713189445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,127,0.04844800134499868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,127,0.04048533240954081
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,255,0.05384533107280731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,255,0.04442666471004486
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,511,0.09212266405423482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,1,0.011813333878914515
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,511,0.06155733267466227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,1,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,3,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,3,0.012096000214417776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,7,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,7,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,15,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,15,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,31,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,31,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,63,0.011690666278203329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,63,0.011658667276302973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,127,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,127,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,255,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,255,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,511,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,float16,1023,0.16895999511082968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,511,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,1023,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,4,2,128,0,1,float16,fp8,1023,0.09554133812586467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,1023,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,2047,0.017071999609470367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,2047,0.016976000120242436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,4095,0.01748266691962878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,4095,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,8191,0.018933333456516266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,8191,0.020714666694402695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,16383,0.024218666056791942
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,16383,0.02160000056028366
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,32767,0.04251733422279358
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,32767,0.026127999027570088
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,65535,0.060922667384147644
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,65535,0.040250666439533234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,1,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,1,0.011999999483426413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,3,0.012026666353146235
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,3,0.011765333513418833
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,7,0.011594666788975397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,7,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,float16,131071,0.10166399677594502
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,15,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,1,128,0,1,float16,fp8,131071,0.06190933287143707
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,15,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,31,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,31,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,63,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,63,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,127,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,255,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,255,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,511,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,511,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,1023,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,1023,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,2047,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,2047,0.015615999698638916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,4095,0.017680000513792038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,4095,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,8191,0.02162133405605952
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,8191,0.019999999552965164
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,16383,0.040048000713189445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,16383,0.025648000339667004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,32767,0.039605334401130676
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,32767,0.06066133578618368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,65535,0.10192533334096272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,65535,0.058543999989827476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,1,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,1,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,1,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,1,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,3,0.010506667196750641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,3,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,3,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,float16,131071,0.18587199846903482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,7,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,7,0.012149333953857422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,7,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,7,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,15,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,15,0.012074666718641916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,15,0.01173866664369901
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,15,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,31,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,31,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,31,0.011930666863918304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,63,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,31,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,4,2,128,0,1,float16,fp8,131071,0.10029866298039754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,63,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,63,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,63,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,127,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,127,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,127,0.012229333321253458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,127,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,255,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,255,0.01020800011853377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,255,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,255,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,511,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,511,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,511,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,511,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,1023,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,1023,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,1023,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,2047,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,2047,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,2047,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,2047,0.014831999937693277
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,4095,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,4095,0.016821333517630894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,4095,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,4095,0.01716800034046173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,8191,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,8191,0.01929066702723503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,8191,0.013455999394257864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,8191,0.01937066639463107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,16383,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,16383,0.025066666305065155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,16383,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,16383,0.023376000424226124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,32767,0.031770666440327965
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,32767,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,32767,0.013418667018413544
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,32767,0.03948266555865606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,65535,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,65535,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,65535,0.05886933207511902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,65535,0.0525439977645874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,float16,131071,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,128,1,float16,fp8,131071,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,float16,131071,0.10354133447011311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,1,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,1,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,1,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,1,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,3,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,3,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,3,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,3,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,7,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,7,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,7,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,7,0.012298667182525
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,15,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,15,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,15,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,31,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,31,0.009872000043590864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,31,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,31,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,63,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,63,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,63,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,63,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,127,0.011744000017642975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,127,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,127,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,255,0.010746666540702185
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,127,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,255,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,255,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,255,0.01163200040658315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,511,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,511,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,511,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,511,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,1023,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,1023,0.010832000523805618
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,1023,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,1023,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,2047,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,2047,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,2047,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,2047,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,64,0,1,float16,fp8,131071,0.08928533395131429
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,4095,0.010591999938090643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,4095,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,4095,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,4095,0.012879999975363413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,8191,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,8191,0.01657066618402799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,8191,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,8191,0.016783999900023144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,16383,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,16383,0.019823999454577763
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,16383,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,16383,0.02107733239730199
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,32767,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,32767,0.028090665737787884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,32767,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,32767,0.02775999903678894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,65535,0.011461333682139715
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,65535,0.0322773332397143
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,65535,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,65535,0.031898667414983116
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,float16,131071,0.03342933456103007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,float16,131071,0.014245333770910898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,128,1,float16,fp8,131071,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,64,0,1,float16,fp8,131071,0.03363200028737386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,1,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,1,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,1,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,3,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,3,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,3,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,7,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,7,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,7,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,7,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,15,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,15,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,31,0.010496000448862711
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,15,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,31,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,31,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,31,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,63,0.010661333799362183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,63,0.010485333700974783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,63,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,63,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,127,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,127,0.010559999694426855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,127,0.010672000547250112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,127,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,255,0.01073066641887029
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,255,0.012346666306257248
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,255,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,511,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,511,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,255,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,511,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,511,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,1023,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,1023,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,1023,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,2047,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,2047,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,2047,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,2047,0.011882666498422623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,4095,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,4095,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,4095,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,4095,0.013562666873137156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,8191,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,8191,0.017551999539136887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,8191,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,8191,0.0173333336909612
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,16383,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,16383,0.021226666867733
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,16383,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,16383,0.01942933350801468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,32767,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,32767,0.023472001155217487
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,32767,0.011695999652147293
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,32767,0.024085332949956257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,65535,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,65535,0.024890666206677754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,65535,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,65535,0.02526933451493581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,float16,131071,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,float16,131071,0.027402666707833607
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,128,1,float16,fp8,131071,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,1,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,1,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,1,0.011855999628702799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,1,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,3,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,3,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,3,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,3,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,7,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,7,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,7,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,7,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,15,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,15,0.010853332777818045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,15,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,15,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,64,0,1,float16,fp8,131071,0.025450666745503742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,31,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,31,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,31,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,31,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,63,0.010816000401973724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,63,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,63,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,63,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,127,0.011050666371981302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,127,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,127,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,255,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,255,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,255,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,255,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,511,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,511,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,511,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,511,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,1023,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,1023,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,1023,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,1023,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,2047,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,2047,0.01725333308180173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,2047,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,2047,0.015568000574906668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,4095,0.013455999394257864
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,4095,0.019178666174411774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,4095,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,4095,0.019082666685183842
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,8191,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,8191,0.023775999744733173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,8191,0.022863999009132385
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,8191,0.013429333766301474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,16383,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,16383,0.0383146678407987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,16383,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,16383,0.03196800003449122
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,32767,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,32767,0.014096000542243322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,32767,0.05906666815280914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,32767,0.05299200117588043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,65535,0.10153599580128987
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,65535,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,65535,0.01350933313369751
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,65535,0.0872373382250468
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,float16,131071,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,float16,131071,0.1850773294766744
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,1,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,1,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,1,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,1,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,3,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,3,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,3,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,3,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,7,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,7,0.01240533341964086
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,7,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,7,0.011584000041087469
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,15,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,15,0.010608000059922537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,15,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,31,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,15,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,31,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,31,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,31,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,63,0.010575999816258749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,128,1,float16,fp8,131071,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,63,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,63,0.01097600037852923
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,127,0.011146667102972666
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,63,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,127,0.010506667196750641
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,127,0.010826667149861654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,255,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,127,0.010768000036478043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,255,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,255,0.010714666297038397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,255,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,511,0.010847999403874079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,511,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,511,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,511,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,1023,0.010656000425418219
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,1023,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,1023,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,1023,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,2047,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,2047,0.013541333377361298
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,2047,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,2047,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,4095,0.013381333400805792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,4095,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,4095,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,4095,0.014298666268587112
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,8191,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,64,0,1,float16,fp8,131071,0.15903466939926147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,8191,0.019082666685183842
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,8191,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,8191,0.019071999937295914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,16383,0.013327999661366144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,16383,0.01974933346112569
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,16383,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,16383,0.021194666624069214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,32767,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,32767,0.022240000466505688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,32767,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,32767,0.021397332350413006
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,65535,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,65535,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,65535,0.023413332800070446
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,65535,0.023317334552605946
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,float16,131071,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,128,1,float16,fp8,131071,0.014959999670584997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,float16,131071,0.029653333127498627
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,1,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,1,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,1,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,1,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,64,0,1,float16,fp8,131071,0.02917333443959554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,3,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,3,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,3,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,3,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,7,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,7,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,7,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,7,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,15,0.010677333921194077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,15,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,15,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,15,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,31,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,31,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,31,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,31,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,63,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,63,0.010933333386977514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,63,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,127,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,127,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,127,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,127,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,255,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,255,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,255,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,255,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,511,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,511,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,511,0.012543999900420507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,511,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,1023,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,1023,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,1023,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,1023,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,2047,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,2047,0.01897066707412402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,2047,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,2047,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,4095,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,4095,0.023813332120577495
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,4095,0.013503999759753546
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,4095,0.02165866643190384
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,8191,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,8191,0.03774933268626531
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,8191,0.013552000125249227
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,8191,0.031632001201311745
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,16383,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,16383,0.06027733286221822
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,16383,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,16383,0.05234666665395101
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,32767,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,32767,0.1018293301264445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,32767,0.01340266689658165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,32767,0.0865280032157898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,float16,65535,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,float16,65535,0.1844586730003357
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,128,1,float16,fp8,65535,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,1,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,1,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,1,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,1,0.013797332843144735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,3,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,64,0,1,float16,fp8,65535,0.1564853290716807
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,3,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,3,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,3,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,7,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,7,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,7,0.013317332913478216
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,7,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,15,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,15,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,15,0.012639999389648438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,15,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,31,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,31,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,31,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,31,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,63,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,63,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,63,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,63,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,127,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,127,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,127,0.01211200033624967
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,127,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,255,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,255,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,255,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,255,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,511,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,511,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,511,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,511,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,1023,0.010911999891201654
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,1023,0.01643199970324834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,1023,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,1023,0.015157333264748255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,2047,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,2047,0.025120000044504803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,2047,0.015546667079130808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,2047,0.023306667804718018
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,4095,0.01481066644191742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,4095,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,4095,0.03976533313592275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,4095,0.031290667752424874
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,8191,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,8191,0.015386667102575302
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,8191,0.06268266836802165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,8191,0.05209066470464071
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,float16,16383,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,float16,16383,0.1051626702149709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,128,1,float16,fp8,16383,0.015439999600251516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,64,0,1,float16,fp8,16383,0.08778132994969685
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,1,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,1,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,1,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,1,0.014741333822409311
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,3,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,3,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,3,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,3,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,7,0.014805333067973455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,7,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,7,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,7,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,15,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,15,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,15,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,15,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,31,0.014890667051076889
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,31,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,31,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,31,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,63,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,63,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,63,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,63,0.013301332791646322
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,127,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,127,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,127,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,127,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,255,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,255,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,255,0.013562666873137156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,255,0.014874666929244995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,511,0.01492799942692121
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,511,0.016965333372354507
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,511,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,511,0.01716800034046173
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,1023,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,1023,0.023200000325838726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,1023,0.021615999440352123
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,1023,0.01569066693385442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,2047,0.017680000513792038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,2047,0.040192000567913055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,2047,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,2047,0.03374933451414108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,4095,0.017845333864291508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,4095,0.062218666076660156
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,4095,0.01766933376590411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,4095,0.054192001620928444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,float16,8191,0.017871999492247898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,float16,8191,0.105295995871226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,1,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,128,1,float16,fp8,8191,0.01766933376590411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,64,0,1,float16,fp8,8191,0.08705066641171773
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,1,0.010879999647537867
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,1,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,1,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,3,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,3,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,3,0.010757333288590113
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,3,0.011648000528415045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,7,0.011551999797423681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,7,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,7,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,7,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,15,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,15,0.010778666784365972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,15,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,31,0.010666667173306147
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,15,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,31,0.010682666053374609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,63,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,31,0.010821333775917688
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,31,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,63,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,63,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,63,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,127,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,127,0.012538666526476542
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,127,0.010837333897749582
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,127,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,255,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,255,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,255,0.010842667271693548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,255,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,511,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,511,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,511,0.01081066702802976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,511,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,1023,0.010863999525705973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,1023,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,1023,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,1023,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,2047,0.012634667257467905
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,2047,0.01469333345691363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,2047,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,2047,0.013807999591032663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,4095,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,4095,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,4095,0.017130666722853977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,4095,0.016869333883126576
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,8191,0.012885333349307379
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,8191,0.018719999740521114
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,8191,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,8191,0.017557332913080852
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,16383,0.012896000097195307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,16383,0.019546666493018467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,16383,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,16383,0.018954666952292126
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,32767,0.021007999777793884
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,32767,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,32767,0.013434667140245438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,32767,0.019765333582957584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,65535,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,65535,0.025600001215934753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,65535,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,65535,0.02555199960867564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,float16,131071,0.04332800209522247
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,float16,131071,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,128,1,float16,fp8,131071,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,1,0.017530667285124462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,1,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,1,0.01756799966096878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,64,0,1,float16,fp8,131071,0.03358400116364161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,1,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,3,0.017610666652520496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,3,0.018405333161354065
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,3,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,3,0.017157333592573803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,7,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,7,0.01783466711640358
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,7,0.01709866647919019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,7,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,15,0.017610666652520496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,15,0.01720533271630605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,15,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,15,0.017157333592573803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,31,0.01766933376590411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,31,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,31,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,31,0.0170666662355264
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,63,0.01739199956258138
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,63,0.017194667210181553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,63,0.017562666287024815
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,63,0.01720533271630605
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,127,0.017258666455745697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,127,0.01709866647919019
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,127,0.01775466650724411
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,127,0.017573333034912746
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,255,0.017642666896184284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,255,0.017210666090250015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,255,0.01743999992807706
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,255,0.017605333278576534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,511,0.017301333447297413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,511,0.021498667697111767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,511,0.017605333278576534
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,511,0.021557333568731945
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,1023,0.017642666896184284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,1023,0.03398400048414866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,1023,0.017184000462293625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,1023,0.029887999097506206
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,2047,0.021327999730904896
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,2047,0.021162666380405426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,2047,0.057215998570124306
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,2047,0.051914667089780174
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,float16,4095,0.021642667551835377
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,float16,4095,0.09551999966303508
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,128,1,float16,fp8,4095,0.019941333681344986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,64,0,1,float16,fp8,4095,0.08213866750399272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,1,0.025589334468046825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,1,0.024853333830833435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,1,0.024197332561016083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,1,0.02603200078010559
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,3,0.02443733314673106
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,3,0.02481599897146225
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,3,0.0240639994541804
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,3,0.027509334186712902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,7,0.025173333783944447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,7,0.024373332659403484
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,7,0.02584533393383026
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,7,0.025802666942278545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,15,0.025120000044504803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,15,0.025802666942278545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,15,0.025370667378107708
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,15,0.023445333043734234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,31,0.025290665527184803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,31,0.02535466601451238
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,31,0.025242666403452556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,31,0.02386666586001714
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,63,0.025674665967623394
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,63,0.025888000925381977
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,63,0.02553066611289978
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,63,0.023919999599456787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,127,0.023589332898457844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,127,0.02532266577084859
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,127,0.02552533398071925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,127,0.02403733382622401
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,255,0.025744001070658367
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,255,0.023520000278949738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,255,0.025578667720158894
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,255,0.024245334168275196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,511,0.025450666745503742
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,511,0.036576000352700554
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,511,0.02624000112215678
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,511,0.0324799989660581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,1023,0.025439999997615814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,1023,0.0547626664241155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,1023,0.02517866591612498
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,1023,0.05023466547330221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,float16,2047,0.028234665592511494
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,float16,2047,0.09821866949399312
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,1,0.011749333391586939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,1,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,1,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,1,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,3,0.0124746672809124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,3,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,3,0.01108266661564509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,3,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,7,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,128,1,float16,fp8,2047,0.027530667682488758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,7,0.012058666596810022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,7,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,7,0.012442667037248611
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,15,0.010751999914646149
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,15,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,15,0.011930666863918304
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,15,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,31,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,31,0.011941333611806234
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,31,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,31,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,63,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,63,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,63,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,63,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,127,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,127,0.010773333410422007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,127,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,127,0.011029332876205444
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,255,0.010677333921194077
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,255,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,255,0.010645333677530289
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,255,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,511,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,511,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,511,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,511,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,1023,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,64,0,1,float16,fp8,2047,0.08501332998275757
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,1023,0.011717333147923151
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,1023,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,1023,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,2047,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,2047,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,2047,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,2047,0.015392000476519266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,4095,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,4095,0.018144000321626663
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,4095,0.013365333278973898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,4095,0.017055999487638474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,8191,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,8191,0.019386666516462963
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,8191,0.013349333157142004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,8191,0.017770666629076004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,16383,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,16383,0.021562665700912476
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,16383,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,16383,0.01956266661485036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,32767,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,32767,0.013637332866589228
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,32767,0.025455998877684276
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,32767,0.02364266663789749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,65535,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,65535,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,65535,0.04013866682847341
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,65535,0.03133866687615713
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,float16,131071,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,1,0.01080000028014183
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,128,1,float16,fp8,131071,0.014949332922697067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,float16,131071,0.060080001751581825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,1,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,64,0,1,float16,fp8,131071,0.054048001766204834
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,3,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,3,0.010858666151762009
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,7,0.010784000158309937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,7,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,15,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,31,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,15,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,31,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,63,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,127,0.01071999967098236
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,127,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,255,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,255,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,511,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,511,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,1023,0.011866666376590729
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,1023,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,2047,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,2047,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,4095,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,4095,0.01732800031701724
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,8191,0.021189334491888683
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,8191,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,16383,0.04035199930270513
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,16383,0.02313599983851115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,32767,0.06083733340104421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,32767,0.03974399964014689
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,65535,0.10195199648539226
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,65535,0.0606879989306132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,1,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,1,0.011744000017642975
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,3,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,3,0.01166933278242747
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,7,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,7,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,15,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,float16,131071,0.18496533234914145
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,15,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,31,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,63,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,31,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,63,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,127,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,127,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,255,0.01163200040658315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,255,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,511,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,511,0.011962667107582092
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,1023,0.011690666278203329
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,1023,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,32,1,2,1,128,0,1,float16,fp8,131071,0.10156266887982686
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,2047,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,2047,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,4095,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,4095,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,8191,0.017893332988023758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,8191,0.01736533393462499
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,16383,0.021082667013009388
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,16383,0.01937066639463107
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,32767,0.02739733209212621
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,32767,0.02749866743882497
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,65535,0.03382933388153712
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,1,0.01062400018175443
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,65535,0.029482667644818623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,1,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,3,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,3,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,7,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,7,0.010794666906197866
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,15,0.010693332801262537
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,15,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,31,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,63,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,31,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,63,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,127,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,127,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,float16,131071,0.03588266670703888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1,1,2,1,128,0,1,float16,fp8,131071,0.03323200096686681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,255,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,255,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,511,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,511,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,1023,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,1023,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,2047,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,2047,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,4095,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,4095,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,8191,0.017024000485738117
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,8191,0.016800000021855038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,16383,0.023077333966890972
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,16383,0.02040533348917961
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,32767,0.024469333390394848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,32767,0.02369066576162974
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,65535,0.02756800005833308
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,65535,0.025600001215934753
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,1,0.010709332923094431
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,3,0.010960000256697336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,1,0.011792000383138657
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,3,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,7,0.010890666395425797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,7,0.01101333275437355
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,15,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,15,0.010874666273593903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,31,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,31,0.011658667276302973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,float16,131071,0.031370667119820915
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,63,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2,1,2,1,128,0,1,float16,fp8,131071,0.02752000093460083
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,63,0.011621333658695221
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,127,0.010629333555698395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,127,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,255,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,255,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,511,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,511,0.012245333443085352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,1023,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,1023,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,2047,0.017317333569129307
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,2047,0.017045332739750545
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,4095,0.021381333470344543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,4095,0.019717333217461903
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,8191,0.03990400085846583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,8191,0.02526933451493581
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,16383,0.0612960010766983
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,16383,0.03940266619126002
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,32767,0.10206933816274007
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,32767,0.05995733539263407
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,65535,0.18262932697931925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,65535,0.09981866677602132
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,1,0.012042666474978128
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,1,0.011770666887362799
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,3,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,3,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,7,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,15,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,7,0.012085333466529846
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,15,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,31,0.011514666179815928
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,31,0.011407999942700068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,63,0.011893333246310553
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,63,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,127,0.011887999872366587
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,127,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,255,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,255,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,511,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,511,0.01414399966597557
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,1023,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,1023,0.013434667140245438
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,2047,0.014842666685581207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,2047,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,4095,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,4095,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,float16,131071,0.34699201583862305
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,8191,0.01926933353145917
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,8191,0.019472000499566395
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,16383,0.02239999920129776
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,16383,0.021520001192887623
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,64,1,2,1,128,0,1,float16,fp8,131071,0.1797013282775879
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,32767,0.023797333240509033
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,32767,0.021509334444999695
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,65535,0.027429332335789997
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,1,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,65535,0.024160000185171764
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,1,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,3,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,3,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,7,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,7,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,15,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,15,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,float16,131071,0.048207998275756836
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,4,1,2,1,128,0,1,float16,fp8,131071,0.02957333376010259
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,31,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,31,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,63,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,63,0.01209066684047381
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,127,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,127,0.01181866725285848
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,255,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,511,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,255,0.011792000383138657
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,511,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,1023,0.014725333700577417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,1023,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,2047,0.021130666136741638
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,2047,0.01897066707412402
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,4095,0.039813332259655
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,4095,0.023914667467276256
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,8191,0.06061333417892456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,8191,0.04001066585381826
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,16383,0.1016373336315155
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,16383,0.05866133173306783
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,32767,0.18231467405954996
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,32767,0.0978613297144572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,1,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,1,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,3,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,3,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,7,0.013514666507641474
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,7,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,15,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,15,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,31,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,31,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,63,0.012719999998807907
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,63,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,127,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,255,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,127,0.013839999834696451
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,fp8,65535,0.17652267217636108
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,128,1,2,1,128,0,1,float16,float16,65535,0.34438931941986084
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,255,0.012794667234023413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,511,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,511,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,1023,0.019141333798567455
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,1023,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,2047,0.041738669077555336
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,2047,0.025642665723959606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,4095,0.06288533409436543
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,4095,0.040762667854626976
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,8191,0.10319999853769939
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,8191,0.06300266583760579
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,1,0.015520000209410986
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,float16,16383,0.18399999539057413
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,1,0.015274666249752045
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,256,1,2,1,128,0,1,float16,fp8,16383,0.1032533347606659
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,3,0.01509333277742068
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,3,0.015439999600251516
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,7,0.01525866612792015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,15,0.01524266724785169
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,7,0.014922666052977243
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,15,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,31,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,31,0.014842666685581207
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,63,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,63,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,127,0.015189333508412043
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,127,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,255,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,255,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,511,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,511,0.019813333948453266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,1023,0.03364799916744232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,1023,0.021850667893886566
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,2047,0.06136000156402588
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,2047,0.041221333046754204
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,4095,0.10381866494814555
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,4095,0.062208001812299095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,1,0.01099733387430509
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,1,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,3,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,3,0.011354666203260422
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,7,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,7,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,15,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,15,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,31,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,31,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,63,0.010949333508809408
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,63,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,127,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,127,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,255,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,255,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,511,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,511,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,float16,8191,0.18759467204411825
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,1023,0.010885333021481832
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,1023,0.011616000284751257
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,2047,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,2047,0.01533866673707962
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,4095,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,4095,0.017125333348910015
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,512,1,2,1,128,0,1,float16,fp8,8191,0.10390399893124898
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,8191,0.019600000232458115
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,8191,0.01932266727089882
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,16383,0.02128000060717265
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,16383,0.021674667795499165
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,32767,0.025392000873883564
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,32767,0.021162666380405426
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,65535,0.04229333500067393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,65535,0.025663999219735462
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,1,0.019280000279347103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,1,0.019248000035683315
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,float16,131071,0.06474133332570393
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,8,1,2,1,128,0,1,float16,fp8,131071,0.042949333786964417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,3,0.01937599976857503
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,3,0.019088000059127808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,7,0.01924266666173935
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,15,0.019733333339293797
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,7,0.017594666530688603
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,15,0.019199999670187633
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,31,0.01956266661485036
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,31,0.01728533332546552
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,63,0.019354666272799175
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,63,0.019343999524911244
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,127,0.01912533367673556
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,127,0.017386666188637417
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,255,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,255,0.017685333887736004
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,511,0.03262399882078171
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,511,0.023237332701683044
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,1023,0.053904001911481224
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,1023,0.034143999218940735
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,2047,0.09715200463930766
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,2047,0.05851200222969055
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,1,0.027530667682488758
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,1,0.025402667621771496
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,float16,4095,0.17502933740615845
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,1024,1,2,1,128,0,1,float16,fp8,4095,0.09525332848230998
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,3,0.0276053324341774
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,3,0.02569599946339925
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,7,0.02899733434120814
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,7,0.02532800038655599
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,15,0.027509334186712902
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,15,0.025498665869235992
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,31,0.029205332199732464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,31,0.0252960001428922
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,63,0.02789866675933202
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,63,0.025562666356563568
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,127,0.02940266579389572
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,127,0.025733334322770435
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,255,0.03389866650104523
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,255,0.02601066728432973
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,511,0.03754133234421412
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,1023,0.09196266531944275
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,1023,0.055258666475613914
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,1,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,1,0.010901333143313726
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,3,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,3,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,7,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,7,0.010944000134865442
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,15,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,15,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,31,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,31,0.010938666760921478
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,63,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,63,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,2047,0.1752799948056539
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,127,0.010928000013033548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,float16,511,0.05407466491063436
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,127,0.010970667004585266
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,255,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,255,0.011594666788975397
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,511,0.012714666624863943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,2048,1,2,1,128,0,1,float16,fp8,2047,0.09664533535639445
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,511,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,1023,0.012373333175977072
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,1023,0.013610667238632837
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,2047,0.01543466622630755
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,2047,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,4095,0.018885333091020584
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,4095,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,8191,0.019637333850065868
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,8191,0.019280000279347103
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,16383,0.025642665723959606
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,16383,0.021354667842388153
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,32767,0.04218666752179464
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,32767,0.025285333395004272
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,65535,0.06206400195757548
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,65535,0.04128533353408178
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,fp8,131071,0.06108800073464712
TRTLLM,1.1.0,NVIDIA GB200,generation_attention,torch_flow,16,1,2,1,128,0,1,float16,float16,131071,0.10135466853777568
